Eric Lee / linux-smarc-t335x-v3.2

Commit be835674b55324c1abe973b15343c3663910c620

Authored by Linus Torvalds 2010-05-13 09:48:26 +0800

Exists in master and in 4 other branches

Merge branch 'merge' of git://git.kernel.org/pub/scm/linux/kernel/git/benh/powerpc

* 'merge' of git://git.kernel.org/pub/scm/linux/kernel/git/benh/powerpc:
  powerpc/perf_event: Fix oops due to perf_event_do_pending call
  powerpc/swiotlb: Fix off by one in determining boundary of which ops to use

Showing 6 changed files Inline Diff

arch/powerpc/include/asm/hw_irq.h
arch/powerpc/kernel/asm-offsets.c
arch/powerpc/kernel/dma-swiotlb.c
arch/powerpc/kernel/entry_64.S
arch/powerpc/kernel/irq.c
arch/powerpc/kernel/time.c

arch/powerpc/include/asm/hw_irq.h

Diff comments View file @ be83567

 /*
  * Copyright (C) 1999 Cort Dougan <cort@cs.nmt.edu>
  */
 #ifndef _ASM_POWERPC_HW_IRQ_H
 #define _ASM_POWERPC_HW_IRQ_H
 #ifdef __KERNEL__
 #include <linux/errno.h>
 #include <linux/compiler.h>
 #include <asm/ptrace.h>
 #include <asm/processor.h>
 extern void timer_interrupt(struct pt_regs *);
 #ifdef CONFIG_PPC64
 #include <asm/paca.h>
 static inline unsigned long local_get_flags(void)
 {
 	unsigned long flags;
 	__asm__ __volatile__("lbz %0,%1(13)"
 	: "=r" (flags)
 	: "i" (offsetof(struct paca_struct, soft_enabled)));
 	return flags;
 }
 static inline unsigned long raw_local_irq_disable(void)
 {
 	unsigned long flags, zero;
 	__asm__ __volatile__("li %1,0; lbz %0,%2(13); stb %1,%2(13)"
 	: "=r" (flags), "=&r" (zero)
 	: "i" (offsetof(struct paca_struct, soft_enabled))
 	: "memory");
 	return flags;
 }
 extern void raw_local_irq_restore(unsigned long);
 extern void iseries_handle_interrupts(void);
 #define raw_local_irq_enable()		raw_local_irq_restore(1)
 #define raw_local_save_flags(flags)	((flags) = local_get_flags())
 #define raw_local_irq_save(flags)	((flags) = raw_local_irq_disable())
 #define raw_irqs_disabled()		(local_get_flags() == 0)
 #define raw_irqs_disabled_flags(flags)	((flags) == 0)
 #ifdef CONFIG_PPC_BOOK3E
 #define __hard_irq_enable()	__asm__ __volatile__("wrteei 1": : :"memory");
 #define __hard_irq_disable()	__asm__ __volatile__("wrteei 0": : :"memory");
 #else
 #define __hard_irq_enable()	__mtmsrd(mfmsr() | MSR_EE, 1)
 #define __hard_irq_disable()	__mtmsrd(mfmsr() & ~MSR_EE, 1)
 #endif
 #define  hard_irq_disable()			\
 	do {					\
 		__hard_irq_disable();		\
 		get_paca()->soft_enabled = 0;	\
 		get_paca()->hard_enabled = 0;	\
 	} while(0)
 #else
 #if defined(CONFIG_BOOKE)
 #define SET_MSR_EE(x)	mtmsr(x)
 #define raw_local_irq_restore(flags)	__asm__ __volatile__("wrtee %0" : : "r" (flags) : "memory")
 #else
 #define SET_MSR_EE(x)	mtmsr(x)
 #define raw_local_irq_restore(flags)	mtmsr(flags)
 #endif
 static inline void raw_local_irq_disable(void)
 {
 #ifdef CONFIG_BOOKE
 	__asm__ __volatile__("wrteei 0": : :"memory");
 #else
 	unsigned long msr;
 	msr = mfmsr();
 	SET_MSR_EE(msr & ~MSR_EE);
 #endif
 }
 static inline void raw_local_irq_enable(void)
 {
 #ifdef CONFIG_BOOKE
 	__asm__ __volatile__("wrteei 1": : :"memory");
 #else
 	unsigned long msr;
 	msr = mfmsr();
 	SET_MSR_EE(msr | MSR_EE);
 #endif
 }
 static inline void raw_local_irq_save_ptr(unsigned long *flags)
 {
 	unsigned long msr;
 	msr = mfmsr();
 	*flags = msr;
 #ifdef CONFIG_BOOKE
 	__asm__ __volatile__("wrteei 0": : :"memory");
 #else
 	SET_MSR_EE(msr & ~MSR_EE);
 #endif
 }
 #define raw_local_save_flags(flags)	((flags) = mfmsr())
 #define raw_local_irq_save(flags)	raw_local_irq_save_ptr(&flags)
 #define raw_irqs_disabled()		((mfmsr() & MSR_EE) == 0)
 #define raw_irqs_disabled_flags(flags)	(((flags) & MSR_EE) == 0)
 #define hard_irq_disable()		raw_local_irq_disable()
 static inline int irqs_disabled_flags(unsigned long flags)
 {
 	return (flags & MSR_EE) == 0;
 }
 #endif /* CONFIG_PPC64 */
 /*
  * interrupt-retrigger: should we handle this via lost interrupts and IPIs
  * or should we not care like we do now ? --BenH.
  */
 struct irq_chip;
-#ifdef CONFIG_PERF_EVENTS
-#ifdef CONFIG_PPC64
-static inline unsigned long test_perf_event_pending(void)
-{
-	unsigned long x;
-	asm volatile("lbz %0,%1(13)"
-		: "=r" (x)
-		: "i" (offsetof(struct paca_struct, perf_event_pending)));
-	return x;
-}
-static inline void set_perf_event_pending(void)
-{
-	asm volatile("stb %0,%1(13)" : :
-		"r" (1),
-		"i" (offsetof(struct paca_struct, perf_event_pending)));
-}
-static inline void clear_perf_event_pending(void)
-{
-	asm volatile("stb %0,%1(13)" : :
-		"r" (0),
-		"i" (offsetof(struct paca_struct, perf_event_pending)));
-}
-#endif /* CONFIG_PPC64 */
-#else  /* CONFIG_PERF_EVENTS */
-static inline unsigned long test_perf_event_pending(void)
-{
-	return 0;
-}
-static inline void clear_perf_event_pending(void) {}
-#endif /* CONFIG_PERF_EVENTS */
 #endif	/* __KERNEL__ */
 #endif	/* _ASM_POWERPC_HW_IRQ_H */

arch/powerpc/kernel/asm-offsets.c

Diff comments View file @ be83567

 /*
  * This program is used to generate definitions needed by
  * assembly language modules.
  *
  * We use the technique used in the OSF Mach kernel code:
  * generate asm statements containing #defines,
  * compile this file to assembler, and then extract the
  * #defines from the assembly-language output.
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  */
 #include <linux/signal.h>
 #include <linux/sched.h>
 #include <linux/kernel.h>
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/types.h>
 #include <linux/mman.h>
 #include <linux/mm.h>
 #include <linux/suspend.h>
 #include <linux/hrtimer.h>
 #ifdef CONFIG_PPC64
 #include <linux/time.h>
 #include <linux/hardirq.h>
 #endif
 #include <linux/kbuild.h>
 #include <asm/io.h>
 #include <asm/page.h>
 #include <asm/pgtable.h>
 #include <asm/processor.h>
 #include <asm/cputable.h>
 #include <asm/thread_info.h>
 #include <asm/rtas.h>
 #include <asm/vdso_datapage.h>
 #ifdef CONFIG_PPC64
 #include <asm/paca.h>
 #include <asm/lppaca.h>
 #include <asm/cache.h>
 #include <asm/compat.h>
 #include <asm/mmu.h>
 #include <asm/hvcall.h>
 #endif
 #ifdef CONFIG_PPC_ISERIES
 #include <asm/iseries/alpaca.h>
 #endif
 #ifdef CONFIG_KVM
 #include <linux/kvm_host.h>
 #endif
 #ifdef CONFIG_PPC32
 #if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
 #include "head_booke.h"
 #endif
 #endif
 #if defined(CONFIG_FSL_BOOKE)
 #include "../mm/mmu_decl.h"
 #endif
 int main(void)
 {
 	DEFINE(THREAD, offsetof(struct task_struct, thread));
 	DEFINE(MM, offsetof(struct task_struct, mm));
 	DEFINE(MMCONTEXTID, offsetof(struct mm_struct, context.id));
 #ifdef CONFIG_PPC64
 	DEFINE(AUDITCONTEXT, offsetof(struct task_struct, audit_context));
 	DEFINE(SIGSEGV, SIGSEGV);
 	DEFINE(NMI_MASK, NMI_MASK);
 #else
 	DEFINE(THREAD_INFO, offsetof(struct task_struct, stack));
 #endif /* CONFIG_PPC64 */
 	DEFINE(KSP, offsetof(struct thread_struct, ksp));
 	DEFINE(KSP_LIMIT, offsetof(struct thread_struct, ksp_limit));
 	DEFINE(PT_REGS, offsetof(struct thread_struct, regs));
 	DEFINE(THREAD_FPEXC_MODE, offsetof(struct thread_struct, fpexc_mode));
 	DEFINE(THREAD_FPR0, offsetof(struct thread_struct, fpr[0]));
 	DEFINE(THREAD_FPSCR, offsetof(struct thread_struct, fpscr));
 #ifdef CONFIG_ALTIVEC
 	DEFINE(THREAD_VR0, offsetof(struct thread_struct, vr[0]));
 	DEFINE(THREAD_VRSAVE, offsetof(struct thread_struct, vrsave));
 	DEFINE(THREAD_VSCR, offsetof(struct thread_struct, vscr));
 	DEFINE(THREAD_USED_VR, offsetof(struct thread_struct, used_vr));
 #endif /* CONFIG_ALTIVEC */
 #ifdef CONFIG_VSX
 	DEFINE(THREAD_VSR0, offsetof(struct thread_struct, fpr));
 	DEFINE(THREAD_USED_VSR, offsetof(struct thread_struct, used_vsr));
 #endif /* CONFIG_VSX */
 #ifdef CONFIG_PPC64
 	DEFINE(KSP_VSID, offsetof(struct thread_struct, ksp_vsid));
 #else /* CONFIG_PPC64 */
 	DEFINE(PGDIR, offsetof(struct thread_struct, pgdir));
 #if defined(CONFIG_4xx) || defined(CONFIG_BOOKE)
 	DEFINE(THREAD_DBCR0, offsetof(struct thread_struct, dbcr0));
 #endif
 #ifdef CONFIG_SPE
 	DEFINE(THREAD_EVR0, offsetof(struct thread_struct, evr[0]));
 	DEFINE(THREAD_ACC, offsetof(struct thread_struct, acc));
 	DEFINE(THREAD_SPEFSCR, offsetof(struct thread_struct, spefscr));
 	DEFINE(THREAD_USED_SPE, offsetof(struct thread_struct, used_spe));
 #endif /* CONFIG_SPE */
 #endif /* CONFIG_PPC64 */
 	DEFINE(TI_FLAGS, offsetof(struct thread_info, flags));
 	DEFINE(TI_LOCAL_FLAGS, offsetof(struct thread_info, local_flags));
 	DEFINE(TI_PREEMPT, offsetof(struct thread_info, preempt_count));
 	DEFINE(TI_TASK, offsetof(struct thread_info, task));
 	DEFINE(TI_CPU, offsetof(struct thread_info, cpu));
 #ifdef CONFIG_PPC64
 	DEFINE(DCACHEL1LINESIZE, offsetof(struct ppc64_caches, dline_size));
 	DEFINE(DCACHEL1LOGLINESIZE, offsetof(struct ppc64_caches, log_dline_size));
 	DEFINE(DCACHEL1LINESPERPAGE, offsetof(struct ppc64_caches, dlines_per_page));
 	DEFINE(ICACHEL1LINESIZE, offsetof(struct ppc64_caches, iline_size));
 	DEFINE(ICACHEL1LOGLINESIZE, offsetof(struct ppc64_caches, log_iline_size));
 	DEFINE(ICACHEL1LINESPERPAGE, offsetof(struct ppc64_caches, ilines_per_page));
 	/* paca */
 	DEFINE(PACA_SIZE, sizeof(struct paca_struct));
 	DEFINE(PACAPACAINDEX, offsetof(struct paca_struct, paca_index));
 	DEFINE(PACAPROCSTART, offsetof(struct paca_struct, cpu_start));
 	DEFINE(PACAKSAVE, offsetof(struct paca_struct, kstack));
 	DEFINE(PACACURRENT, offsetof(struct paca_struct, __current));
 	DEFINE(PACASAVEDMSR, offsetof(struct paca_struct, saved_msr));
 	DEFINE(PACASTABRR, offsetof(struct paca_struct, stab_rr));
 	DEFINE(PACAR1, offsetof(struct paca_struct, saved_r1));
 	DEFINE(PACATOC, offsetof(struct paca_struct, kernel_toc));
 	DEFINE(PACAKBASE, offsetof(struct paca_struct, kernelbase));
 	DEFINE(PACAKMSR, offsetof(struct paca_struct, kernel_msr));
 	DEFINE(PACASOFTIRQEN, offsetof(struct paca_struct, soft_enabled));
 	DEFINE(PACAHARDIRQEN, offsetof(struct paca_struct, hard_enabled));
-	DEFINE(PACAPERFPEND, offsetof(struct paca_struct, perf_event_pending));
 	DEFINE(PACACONTEXTID, offsetof(struct paca_struct, context.id));
 #ifdef CONFIG_PPC_MM_SLICES
 	DEFINE(PACALOWSLICESPSIZE, offsetof(struct paca_struct,
 					    context.low_slices_psize));
 	DEFINE(PACAHIGHSLICEPSIZE, offsetof(struct paca_struct,
 					    context.high_slices_psize));
 	DEFINE(MMUPSIZEDEFSIZE, sizeof(struct mmu_psize_def));
 #endif /* CONFIG_PPC_MM_SLICES */
 #ifdef CONFIG_PPC_BOOK3E
 	DEFINE(PACAPGD, offsetof(struct paca_struct, pgd));
 	DEFINE(PACA_KERNELPGD, offsetof(struct paca_struct, kernel_pgd));
 	DEFINE(PACA_EXGEN, offsetof(struct paca_struct, exgen));
 	DEFINE(PACA_EXTLB, offsetof(struct paca_struct, extlb));
 	DEFINE(PACA_EXMC, offsetof(struct paca_struct, exmc));
 	DEFINE(PACA_EXCRIT, offsetof(struct paca_struct, excrit));
 	DEFINE(PACA_EXDBG, offsetof(struct paca_struct, exdbg));
 	DEFINE(PACA_MC_STACK, offsetof(struct paca_struct, mc_kstack));
 	DEFINE(PACA_CRIT_STACK, offsetof(struct paca_struct, crit_kstack));
 	DEFINE(PACA_DBG_STACK, offsetof(struct paca_struct, dbg_kstack));
 #endif /* CONFIG_PPC_BOOK3E */
 #ifdef CONFIG_PPC_STD_MMU_64
 	DEFINE(PACASTABREAL, offsetof(struct paca_struct, stab_real));
 	DEFINE(PACASTABVIRT, offsetof(struct paca_struct, stab_addr));
 	DEFINE(PACASLBCACHE, offsetof(struct paca_struct, slb_cache));
 	DEFINE(PACASLBCACHEPTR, offsetof(struct paca_struct, slb_cache_ptr));
 	DEFINE(PACAVMALLOCSLLP, offsetof(struct paca_struct, vmalloc_sllp));
 #ifdef CONFIG_PPC_MM_SLICES
 	DEFINE(MMUPSIZESLLP, offsetof(struct mmu_psize_def, sllp));
 #else
 	DEFINE(PACACONTEXTSLLP, offsetof(struct paca_struct, context.sllp));
 #endif /* CONFIG_PPC_MM_SLICES */
 	DEFINE(PACA_EXGEN, offsetof(struct paca_struct, exgen));
 	DEFINE(PACA_EXMC, offsetof(struct paca_struct, exmc));
 	DEFINE(PACA_EXSLB, offsetof(struct paca_struct, exslb));
 	DEFINE(PACALPPACAPTR, offsetof(struct paca_struct, lppaca_ptr));
 	DEFINE(PACA_SLBSHADOWPTR, offsetof(struct paca_struct, slb_shadow_ptr));
 	DEFINE(SLBSHADOW_STACKVSID,
 	       offsetof(struct slb_shadow, save_area[SLB_NUM_BOLTED - 1].vsid));
 	DEFINE(SLBSHADOW_STACKESID,
 	       offsetof(struct slb_shadow, save_area[SLB_NUM_BOLTED - 1].esid));
 	DEFINE(LPPACASRR0, offsetof(struct lppaca, saved_srr0));
 	DEFINE(LPPACASRR1, offsetof(struct lppaca, saved_srr1));
 	DEFINE(LPPACAANYINT, offsetof(struct lppaca, int_dword.any_int));
 	DEFINE(LPPACADECRINT, offsetof(struct lppaca, int_dword.fields.decr_int));
 	DEFINE(SLBSHADOW_SAVEAREA, offsetof(struct slb_shadow, save_area));
 #endif /* CONFIG_PPC_STD_MMU_64 */
 	DEFINE(PACAEMERGSP, offsetof(struct paca_struct, emergency_sp));
 	DEFINE(PACAHWCPUID, offsetof(struct paca_struct, hw_cpu_id));
 	DEFINE(PACA_STARTPURR, offsetof(struct paca_struct, startpurr));
 	DEFINE(PACA_STARTSPURR, offsetof(struct paca_struct, startspurr));
 	DEFINE(PACA_USER_TIME, offsetof(struct paca_struct, user_time));
 	DEFINE(PACA_SYSTEM_TIME, offsetof(struct paca_struct, system_time));
 	DEFINE(PACA_DATA_OFFSET, offsetof(struct paca_struct, data_offset));
 	DEFINE(PACA_TRAP_SAVE, offsetof(struct paca_struct, trap_save));
 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 	DEFINE(PACA_KVM_IN_GUEST, offsetof(struct paca_struct, kvm_in_guest));
 	DEFINE(PACA_KVM_SLB, offsetof(struct paca_struct, kvm_slb));
 	DEFINE(PACA_KVM_SLB_MAX, offsetof(struct paca_struct, kvm_slb_max));
 	DEFINE(PACA_KVM_CR, offsetof(struct paca_struct, shadow_vcpu.cr));
 	DEFINE(PACA_KVM_XER, offsetof(struct paca_struct, shadow_vcpu.xer));
 	DEFINE(PACA_KVM_R0, offsetof(struct paca_struct, shadow_vcpu.gpr[0]));
 	DEFINE(PACA_KVM_R1, offsetof(struct paca_struct, shadow_vcpu.gpr[1]));
 	DEFINE(PACA_KVM_R2, offsetof(struct paca_struct, shadow_vcpu.gpr[2]));
 	DEFINE(PACA_KVM_R3, offsetof(struct paca_struct, shadow_vcpu.gpr[3]));
 	DEFINE(PACA_KVM_R4, offsetof(struct paca_struct, shadow_vcpu.gpr[4]));
 	DEFINE(PACA_KVM_R5, offsetof(struct paca_struct, shadow_vcpu.gpr[5]));
 	DEFINE(PACA_KVM_R6, offsetof(struct paca_struct, shadow_vcpu.gpr[6]));
 	DEFINE(PACA_KVM_R7, offsetof(struct paca_struct, shadow_vcpu.gpr[7]));
 	DEFINE(PACA_KVM_R8, offsetof(struct paca_struct, shadow_vcpu.gpr[8]));
 	DEFINE(PACA_KVM_R9, offsetof(struct paca_struct, shadow_vcpu.gpr[9]));
 	DEFINE(PACA_KVM_R10, offsetof(struct paca_struct, shadow_vcpu.gpr[10]));
 	DEFINE(PACA_KVM_R11, offsetof(struct paca_struct, shadow_vcpu.gpr[11]));
 	DEFINE(PACA_KVM_R12, offsetof(struct paca_struct, shadow_vcpu.gpr[12]));
 	DEFINE(PACA_KVM_R13, offsetof(struct paca_struct, shadow_vcpu.gpr[13]));
 	DEFINE(PACA_KVM_HOST_R1, offsetof(struct paca_struct, shadow_vcpu.host_r1));
 	DEFINE(PACA_KVM_HOST_R2, offsetof(struct paca_struct, shadow_vcpu.host_r2));
 	DEFINE(PACA_KVM_VMHANDLER, offsetof(struct paca_struct,
 					    shadow_vcpu.vmhandler));
 	DEFINE(PACA_KVM_SCRATCH0, offsetof(struct paca_struct,
 					   shadow_vcpu.scratch0));
 	DEFINE(PACA_KVM_SCRATCH1, offsetof(struct paca_struct,
 					   shadow_vcpu.scratch1));
 #endif
 #endif /* CONFIG_PPC64 */
 	/* RTAS */
 	DEFINE(RTASBASE, offsetof(struct rtas_t, base));
 	DEFINE(RTASENTRY, offsetof(struct rtas_t, entry));
 	/* Interrupt register frame */
 	DEFINE(STACK_FRAME_OVERHEAD, STACK_FRAME_OVERHEAD);
 	DEFINE(INT_FRAME_SIZE, STACK_INT_FRAME_SIZE);
 #ifdef CONFIG_PPC64
 	DEFINE(SWITCH_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs));
 	/* Create extra stack space for SRR0 and SRR1 when calling prom/rtas. */
 	DEFINE(PROM_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs) + 16);
 	DEFINE(RTAS_FRAME_SIZE, STACK_FRAME_OVERHEAD + sizeof(struct pt_regs) + 16);
 	/* hcall statistics */
 	DEFINE(HCALL_STAT_SIZE, sizeof(struct hcall_stats));
 	DEFINE(HCALL_STAT_CALLS, offsetof(struct hcall_stats, num_calls));
 	DEFINE(HCALL_STAT_TB, offsetof(struct hcall_stats, tb_total));
 	DEFINE(HCALL_STAT_PURR, offsetof(struct hcall_stats, purr_total));
 #endif /* CONFIG_PPC64 */
 	DEFINE(GPR0, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[0]));
 	DEFINE(GPR1, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[1]));
 	DEFINE(GPR2, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[2]));
 	DEFINE(GPR3, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[3]));
 	DEFINE(GPR4, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[4]));
 	DEFINE(GPR5, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[5]));
 	DEFINE(GPR6, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[6]));
 	DEFINE(GPR7, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[7]));
 	DEFINE(GPR8, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[8]));
 	DEFINE(GPR9, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[9]));
 	DEFINE(GPR10, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[10]));
 	DEFINE(GPR11, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[11]));
 	DEFINE(GPR12, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[12]));
 	DEFINE(GPR13, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[13]));
 #ifndef CONFIG_PPC64
 	DEFINE(GPR14, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[14]));
 	DEFINE(GPR15, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[15]));
 	DEFINE(GPR16, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[16]));
 	DEFINE(GPR17, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[17]));
 	DEFINE(GPR18, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[18]));
 	DEFINE(GPR19, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[19]));
 	DEFINE(GPR20, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[20]));
 	DEFINE(GPR21, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[21]));
 	DEFINE(GPR22, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[22]));
 	DEFINE(GPR23, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[23]));
 	DEFINE(GPR24, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[24]));
 	DEFINE(GPR25, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[25]));
 	DEFINE(GPR26, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[26]));
 	DEFINE(GPR27, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[27]));
 	DEFINE(GPR28, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[28]));
 	DEFINE(GPR29, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[29]));
 	DEFINE(GPR30, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[30]));
 	DEFINE(GPR31, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, gpr[31]));
 #endif /* CONFIG_PPC64 */
 	/*
 	 * Note: these symbols include _ because they overlap with special
 	 * register names
 	 */
 	DEFINE(_NIP, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, nip));
 	DEFINE(_MSR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, msr));
 	DEFINE(_CTR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, ctr));
 	DEFINE(_LINK, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, link));
 	DEFINE(_CCR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, ccr));
 	DEFINE(_XER, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, xer));
 	DEFINE(_DAR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, dar));
 	DEFINE(_DSISR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, dsisr));
 	DEFINE(ORIG_GPR3, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, orig_gpr3));
 	DEFINE(RESULT, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, result));
 	DEFINE(_TRAP, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, trap));
 #ifndef CONFIG_PPC64
 	DEFINE(_MQ, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, mq));
 	/*
 	 * The PowerPC 400-class & Book-E processors have neither the DAR
 	 * nor the DSISR SPRs. Hence, we overload them to hold the similar
 	 * DEAR and ESR SPRs for such processors.  For critical interrupts
 	 * we use them to hold SRR0 and SRR1.
 	 */
 	DEFINE(_DEAR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, dar));
 	DEFINE(_ESR, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, dsisr));
 #else /* CONFIG_PPC64 */
 	DEFINE(SOFTE, STACK_FRAME_OVERHEAD+offsetof(struct pt_regs, softe));
 	/* These _only_ to be used with {PROM,RTAS}_FRAME_SIZE!!! */
 	DEFINE(_SRR0, STACK_FRAME_OVERHEAD+sizeof(struct pt_regs));
 	DEFINE(_SRR1, STACK_FRAME_OVERHEAD+sizeof(struct pt_regs)+8);
 #endif /* CONFIG_PPC64 */
 #if defined(CONFIG_PPC32)
 #if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
 	DEFINE(EXC_LVL_SIZE, STACK_EXC_LVL_FRAME_SIZE);
 	DEFINE(MAS0, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, mas0));
 	/* we overload MMUCR for 44x on MAS0 since they are mutually exclusive */
 	DEFINE(MMUCR, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, mas0));
 	DEFINE(MAS1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, mas1));
 	DEFINE(MAS2, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, mas2));
 	DEFINE(MAS3, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, mas3));
 	DEFINE(MAS6, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, mas6));
 	DEFINE(MAS7, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, mas7));
 	DEFINE(_SRR0, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, srr0));
 	DEFINE(_SRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, srr1));
 	DEFINE(_CSRR0, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, csrr0));
 	DEFINE(_CSRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, csrr1));
 	DEFINE(_DSRR0, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, dsrr0));
 	DEFINE(_DSRR1, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, dsrr1));
 	DEFINE(SAVED_KSP_LIMIT, STACK_INT_FRAME_SIZE+offsetof(struct exception_regs, saved_ksp_limit));
 #endif
 #endif
 	DEFINE(CLONE_VM, CLONE_VM);
 	DEFINE(CLONE_UNTRACED, CLONE_UNTRACED);
 #ifndef CONFIG_PPC64
 	DEFINE(MM_PGD, offsetof(struct mm_struct, pgd));
 #endif /* ! CONFIG_PPC64 */
 	/* About the CPU features table */
 	DEFINE(CPU_SPEC_FEATURES, offsetof(struct cpu_spec, cpu_features));
 	DEFINE(CPU_SPEC_SETUP, offsetof(struct cpu_spec, cpu_setup));
 	DEFINE(CPU_SPEC_RESTORE, offsetof(struct cpu_spec, cpu_restore));
 	DEFINE(pbe_address, offsetof(struct pbe, address));
 	DEFINE(pbe_orig_address, offsetof(struct pbe, orig_address));
 	DEFINE(pbe_next, offsetof(struct pbe, next));
 #ifndef CONFIG_PPC64
 	DEFINE(TASK_SIZE, TASK_SIZE);
 	DEFINE(NUM_USER_SEGMENTS, TASK_SIZE>>28);
 #endif /* ! CONFIG_PPC64 */
 	/* datapage offsets for use by vdso */
 	DEFINE(CFG_TB_ORIG_STAMP, offsetof(struct vdso_data, tb_orig_stamp));
 	DEFINE(CFG_TB_TICKS_PER_SEC, offsetof(struct vdso_data, tb_ticks_per_sec));
 	DEFINE(CFG_TB_TO_XS, offsetof(struct vdso_data, tb_to_xs));
 	DEFINE(CFG_STAMP_XSEC, offsetof(struct vdso_data, stamp_xsec));
 	DEFINE(CFG_TB_UPDATE_COUNT, offsetof(struct vdso_data, tb_update_count));
 	DEFINE(CFG_TZ_MINUTEWEST, offsetof(struct vdso_data, tz_minuteswest));
 	DEFINE(CFG_TZ_DSTTIME, offsetof(struct vdso_data, tz_dsttime));
 	DEFINE(CFG_SYSCALL_MAP32, offsetof(struct vdso_data, syscall_map_32));
 	DEFINE(WTOM_CLOCK_SEC, offsetof(struct vdso_data, wtom_clock_sec));
 	DEFINE(WTOM_CLOCK_NSEC, offsetof(struct vdso_data, wtom_clock_nsec));
 	DEFINE(STAMP_XTIME, offsetof(struct vdso_data, stamp_xtime));
 	DEFINE(CFG_ICACHE_BLOCKSZ, offsetof(struct vdso_data, icache_block_size));
 	DEFINE(CFG_DCACHE_BLOCKSZ, offsetof(struct vdso_data, dcache_block_size));
 	DEFINE(CFG_ICACHE_LOGBLOCKSZ, offsetof(struct vdso_data, icache_log_block_size));
 	DEFINE(CFG_DCACHE_LOGBLOCKSZ, offsetof(struct vdso_data, dcache_log_block_size));
 #ifdef CONFIG_PPC64
 	DEFINE(CFG_SYSCALL_MAP64, offsetof(struct vdso_data, syscall_map_64));
 	DEFINE(TVAL64_TV_SEC, offsetof(struct timeval, tv_sec));
 	DEFINE(TVAL64_TV_USEC, offsetof(struct timeval, tv_usec));
 	DEFINE(TVAL32_TV_SEC, offsetof(struct compat_timeval, tv_sec));
 	DEFINE(TVAL32_TV_USEC, offsetof(struct compat_timeval, tv_usec));
 	DEFINE(TSPC64_TV_SEC, offsetof(struct timespec, tv_sec));
 	DEFINE(TSPC64_TV_NSEC, offsetof(struct timespec, tv_nsec));
 	DEFINE(TSPC32_TV_SEC, offsetof(struct compat_timespec, tv_sec));
 	DEFINE(TSPC32_TV_NSEC, offsetof(struct compat_timespec, tv_nsec));
 #else
 	DEFINE(TVAL32_TV_SEC, offsetof(struct timeval, tv_sec));
 	DEFINE(TVAL32_TV_USEC, offsetof(struct timeval, tv_usec));
 	DEFINE(TSPC32_TV_SEC, offsetof(struct timespec, tv_sec));
 	DEFINE(TSPC32_TV_NSEC, offsetof(struct timespec, tv_nsec));
 #endif
 	/* timeval/timezone offsets for use by vdso */
 	DEFINE(TZONE_TZ_MINWEST, offsetof(struct timezone, tz_minuteswest));
 	DEFINE(TZONE_TZ_DSTTIME, offsetof(struct timezone, tz_dsttime));
 	/* Other bits used by the vdso */
 	DEFINE(CLOCK_REALTIME, CLOCK_REALTIME);
 	DEFINE(CLOCK_MONOTONIC, CLOCK_MONOTONIC);
 	DEFINE(NSEC_PER_SEC, NSEC_PER_SEC);
 	DEFINE(CLOCK_REALTIME_RES, MONOTONIC_RES_NSEC);
 #ifdef CONFIG_BUG
 	DEFINE(BUG_ENTRY_SIZE, sizeof(struct bug_entry));
 #endif
 #ifdef CONFIG_PPC_ISERIES
 	/* the assembler miscalculates the VSID values */
 	DEFINE(PAGE_OFFSET_ESID, GET_ESID(PAGE_OFFSET));
 	DEFINE(PAGE_OFFSET_VSID, KERNEL_VSID(PAGE_OFFSET));
 	DEFINE(VMALLOC_START_ESID, GET_ESID(VMALLOC_START));
 	DEFINE(VMALLOC_START_VSID, KERNEL_VSID(VMALLOC_START));
 	/* alpaca */
 	DEFINE(ALPACA_SIZE, sizeof(struct alpaca));
 #endif
 	DEFINE(PGD_TABLE_SIZE, PGD_TABLE_SIZE);
 	DEFINE(PTE_SIZE, sizeof(pte_t));
 #ifdef CONFIG_KVM
 	DEFINE(VCPU_HOST_STACK, offsetof(struct kvm_vcpu, arch.host_stack));
 	DEFINE(VCPU_HOST_PID, offsetof(struct kvm_vcpu, arch.host_pid));
 	DEFINE(VCPU_GPRS, offsetof(struct kvm_vcpu, arch.gpr));
 	DEFINE(VCPU_LR, offsetof(struct kvm_vcpu, arch.lr));
 	DEFINE(VCPU_CTR, offsetof(struct kvm_vcpu, arch.ctr));
 	DEFINE(VCPU_PC, offsetof(struct kvm_vcpu, arch.pc));
 	DEFINE(VCPU_MSR, offsetof(struct kvm_vcpu, arch.msr));
 	DEFINE(VCPU_SPRG4, offsetof(struct kvm_vcpu, arch.sprg4));
 	DEFINE(VCPU_SPRG5, offsetof(struct kvm_vcpu, arch.sprg5));
 	DEFINE(VCPU_SPRG6, offsetof(struct kvm_vcpu, arch.sprg6));
 	DEFINE(VCPU_SPRG7, offsetof(struct kvm_vcpu, arch.sprg7));
 	DEFINE(VCPU_SHADOW_PID, offsetof(struct kvm_vcpu, arch.shadow_pid));
 	DEFINE(VCPU_LAST_INST, offsetof(struct kvm_vcpu, arch.last_inst));
 	DEFINE(VCPU_FAULT_DEAR, offsetof(struct kvm_vcpu, arch.fault_dear));
 	DEFINE(VCPU_FAULT_ESR, offsetof(struct kvm_vcpu, arch.fault_esr));
 	/* book3s_64 */
 #ifdef CONFIG_PPC64
 	DEFINE(VCPU_FAULT_DSISR, offsetof(struct kvm_vcpu, arch.fault_dsisr));
 	DEFINE(VCPU_HOST_RETIP, offsetof(struct kvm_vcpu, arch.host_retip));
 	DEFINE(VCPU_HOST_R2, offsetof(struct kvm_vcpu, arch.host_r2));
 	DEFINE(VCPU_HOST_MSR, offsetof(struct kvm_vcpu, arch.host_msr));
 	DEFINE(VCPU_SHADOW_MSR, offsetof(struct kvm_vcpu, arch.shadow_msr));
 	DEFINE(VCPU_SHADOW_SRR1, offsetof(struct kvm_vcpu, arch.shadow_srr1));
 	DEFINE(VCPU_TRAMPOLINE_LOWMEM, offsetof(struct kvm_vcpu, arch.trampoline_lowmem));
 	DEFINE(VCPU_TRAMPOLINE_ENTER, offsetof(struct kvm_vcpu, arch.trampoline_enter));
 	DEFINE(VCPU_HIGHMEM_HANDLER, offsetof(struct kvm_vcpu, arch.highmem_handler));
 	DEFINE(VCPU_RMCALL, offsetof(struct kvm_vcpu, arch.rmcall));
 	DEFINE(VCPU_HFLAGS, offsetof(struct kvm_vcpu, arch.hflags));
 #else
 	DEFINE(VCPU_CR, offsetof(struct kvm_vcpu, arch.cr));
 	DEFINE(VCPU_XER, offsetof(struct kvm_vcpu, arch.xer));
 #endif /* CONFIG_PPC64 */
 #endif
 #ifdef CONFIG_44x
 	DEFINE(PGD_T_LOG2, PGD_T_LOG2);
 	DEFINE(PTE_T_LOG2, PTE_T_LOG2);
 #endif
 #ifdef CONFIG_KVM_EXIT_TIMING
 	DEFINE(VCPU_TIMING_EXIT_TBU, offsetof(struct kvm_vcpu,
 						arch.timing_exit.tv32.tbu));
 	DEFINE(VCPU_TIMING_EXIT_TBL, offsetof(struct kvm_vcpu,
 						arch.timing_exit.tv32.tbl));
 	DEFINE(VCPU_TIMING_LAST_ENTER_TBU, offsetof(struct kvm_vcpu,
 					arch.timing_last_enter.tv32.tbu));
 	DEFINE(VCPU_TIMING_LAST_ENTER_TBL, offsetof(struct kvm_vcpu,
 					arch.timing_last_enter.tv32.tbl));
 #endif
 	return 0;
 }

arch/powerpc/kernel/dma-swiotlb.c

Diff comments View file @ be83567

 /*
  * Contains routines needed to support swiotlb for ppc.
  *
- * Copyright (C) 2009 Becky Bruce, Freescale Semiconductor
+ * Copyright (C) 2009-2010 Freescale Semiconductor, Inc.
+ * Author: Becky Bruce
  *
  * This program is free software; you can redistribute  it and/or modify it
  * under  the terms of  the GNU General  Public License as published by the
  * Free Software Foundation;  either version 2 of the  License, or (at your
  * option) any later version.
  *
  */
 #include <linux/dma-mapping.h>
 #include <linux/pfn.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
 #include <linux/pci.h>
 #include <asm/machdep.h>
 #include <asm/swiotlb.h>
 #include <asm/dma.h>
 #include <asm/abs_addr.h>
 unsigned int ppc_swiotlb_enable;
 /*
  * At the moment, all platforms that use this code only require
  * swiotlb to be used if we're operating on HIGHMEM.  Since
  * we don't ever call anything other than map_sg, unmap_sg,
  * map_page, and unmap_page on highmem, use normal dma_ops
  * for everything else.
  */
 struct dma_map_ops swiotlb_dma_ops = {
 	.alloc_coherent = dma_direct_alloc_coherent,
 	.free_coherent = dma_direct_free_coherent,
 	.map_sg = swiotlb_map_sg_attrs,
 	.unmap_sg = swiotlb_unmap_sg_attrs,
 	.dma_supported = swiotlb_dma_supported,
 	.map_page = swiotlb_map_page,
 	.unmap_page = swiotlb_unmap_page,
 	.sync_single_range_for_cpu = swiotlb_sync_single_range_for_cpu,
 	.sync_single_range_for_device = swiotlb_sync_single_range_for_device,
 	.sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
 	.sync_sg_for_device = swiotlb_sync_sg_for_device,
 	.mapping_error = swiotlb_dma_mapping_error,
 };
 void pci_dma_dev_setup_swiotlb(struct pci_dev *pdev)
 {
 	struct pci_controller *hose;
 	struct dev_archdata *sd;
 	hose = pci_bus_to_host(pdev->bus);
 	sd = &pdev->dev.archdata;
 	sd->max_direct_dma_addr =
 		hose->dma_window_base_cur + hose->dma_window_size;
 }
 static int ppc_swiotlb_bus_notify(struct notifier_block *nb,
 				  unsigned long action, void *data)
 {
 	struct device *dev = data;
 	struct dev_archdata *sd;
 	/* We are only intereted in device addition */
 	if (action != BUS_NOTIFY_ADD_DEVICE)
 		return 0;
 	sd = &dev->archdata;
 	sd->max_direct_dma_addr = 0;
 	/* May need to bounce if the device can't address all of DRAM */
-	if (dma_get_mask(dev) < lmb_end_of_DRAM())
+	if ((dma_get_mask(dev) + 1) < lmb_end_of_DRAM())
 		set_dma_ops(dev, &swiotlb_dma_ops);
 	return NOTIFY_DONE;
 }
 static struct notifier_block ppc_swiotlb_plat_bus_notifier = {
 	.notifier_call = ppc_swiotlb_bus_notify,
 	.priority = 0,
 };
 static struct notifier_block ppc_swiotlb_of_bus_notifier = {
 	.notifier_call = ppc_swiotlb_bus_notify,
 	.priority = 0,
 };
 int __init swiotlb_setup_bus_notifier(void)
 {
 	bus_register_notifier(&platform_bus_type,
 			      &ppc_swiotlb_plat_bus_notifier);
 	bus_register_notifier(&of_platform_bus_type,
 			      &ppc_swiotlb_of_bus_notifier);
 	return 0;
 }

arch/powerpc/kernel/entry_64.S

Diff comments View file @ be83567

 /*
  *  PowerPC version
  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
  *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
  *  Adapted for Power Macintosh by Paul Mackerras.
  *  Low-level exception handlers and MMU support
  *  rewritten by Paul Mackerras.
  *    Copyright (C) 1996 Paul Mackerras.
  *  MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
  *
  *  This file contains the system call entry code, context switch
  *  code, and exception/interrupt return code for PowerPC.
  *
  *  This program is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU General Public License
  *  as published by the Free Software Foundation; either version
  *  2 of the License, or (at your option) any later version.
  */
 #include <linux/errno.h>
 #include <asm/unistd.h>
 #include <asm/processor.h>
 #include <asm/page.h>
 #include <asm/mmu.h>
 #include <asm/thread_info.h>
 #include <asm/ppc_asm.h>
 #include <asm/asm-offsets.h>
 #include <asm/cputable.h>
 #include <asm/firmware.h>
 #include <asm/bug.h>
 #include <asm/ptrace.h>
 #include <asm/irqflags.h>
 #include <asm/ftrace.h>
 /*
  * System calls.
  */
 	.section	".toc","aw"
 .SYS_CALL_TABLE:
 	.tc .sys_call_table[TC],.sys_call_table
 /* This value is used to mark exception frames on the stack. */
 exception_marker:
 	.tc	ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
 	.section	".text"
 	.align 7
 #undef SHOW_SYSCALLS
 	.globl system_call_common
 system_call_common:
 	andi.	r10,r12,MSR_PR
 	mr	r10,r1
 	addi	r1,r1,-INT_FRAME_SIZE
 	beq-	1f
 	ld	r1,PACAKSAVE(r13)
 1:	std	r10,0(r1)
 	std	r11,_NIP(r1)
 	std	r12,_MSR(r1)
 	std	r0,GPR0(r1)
 	std	r10,GPR1(r1)
 	ACCOUNT_CPU_USER_ENTRY(r10, r11)
 	/*
 	 * This "crclr so" clears CR0.SO, which is the error indication on
 	 * return from this system call.  There must be no cmp instruction
 	 * between it and the "mfcr r9" below, otherwise if XER.SO is set,
 	 * CR0.SO will get set, causing all system calls to appear to fail.
 	 */
 	crclr	so
 	std	r2,GPR2(r1)
 	std	r3,GPR3(r1)
 	std	r4,GPR4(r1)
 	std	r5,GPR5(r1)
 	std	r6,GPR6(r1)
 	std	r7,GPR7(r1)
 	std	r8,GPR8(r1)
 	li	r11,0
 	std	r11,GPR9(r1)
 	std	r11,GPR10(r1)
 	std	r11,GPR11(r1)
 	std	r11,GPR12(r1)
 	std	r9,GPR13(r1)
 	mfcr	r9
 	mflr	r10
 	li	r11,0xc01
 	std	r9,_CCR(r1)
 	std	r10,_LINK(r1)
 	std	r11,_TRAP(r1)
 	mfxer	r9
 	mfctr	r10
 	std	r9,_XER(r1)
 	std	r10,_CTR(r1)
 	std	r3,ORIG_GPR3(r1)
 	ld	r2,PACATOC(r13)
 	addi	r9,r1,STACK_FRAME_OVERHEAD
 	ld	r11,exception_marker@toc(r2)
 	std	r11,-16(r9)		/* "regshere" marker */
 #ifdef CONFIG_TRACE_IRQFLAGS
 	bl	.trace_hardirqs_on
 	REST_GPR(0,r1)
 	REST_4GPRS(3,r1)
 	REST_2GPRS(7,r1)
 	addi	r9,r1,STACK_FRAME_OVERHEAD
 	ld	r12,_MSR(r1)
 #endif /* CONFIG_TRACE_IRQFLAGS */
 	li	r10,1
 	stb	r10,PACASOFTIRQEN(r13)
 	stb	r10,PACAHARDIRQEN(r13)
 	std	r10,SOFTE(r1)
 #ifdef CONFIG_PPC_ISERIES
 BEGIN_FW_FTR_SECTION
 	/* Hack for handling interrupts when soft-enabling on iSeries */
 	cmpdi	cr1,r0,0x5555		/* syscall 0x5555 */
 	andi.	r10,r12,MSR_PR		/* from kernel */
 	crand	4*cr0+eq,4*cr1+eq,4*cr0+eq
 	bne	2f
 	b	hardware_interrupt_entry
 2:
 END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif /* CONFIG_PPC_ISERIES */
 	/* Hard enable interrupts */
 #ifdef CONFIG_PPC_BOOK3E
 	wrteei	1
 #else
 	mfmsr	r11
 	ori	r11,r11,MSR_EE
 	mtmsrd	r11,1
 #endif /* CONFIG_PPC_BOOK3E */
 #ifdef SHOW_SYSCALLS
 	bl	.do_show_syscall
 	REST_GPR(0,r1)
 	REST_4GPRS(3,r1)
 	REST_2GPRS(7,r1)
 	addi	r9,r1,STACK_FRAME_OVERHEAD
 #endif
 	clrrdi	r11,r1,THREAD_SHIFT
 	ld	r10,TI_FLAGS(r11)
 	andi.	r11,r10,_TIF_SYSCALL_T_OR_A
 	bne-	syscall_dotrace
 syscall_dotrace_cont:
 	cmpldi	0,r0,NR_syscalls
 	bge-	syscall_enosys
 system_call:			/* label this so stack traces look sane */
 /*
  * Need to vector to 32 Bit or default sys_call_table here,
  * based on caller's run-mode / personality.
  */
 	ld	r11,.SYS_CALL_TABLE@toc(2)
 	andi.	r10,r10,_TIF_32BIT
 	beq	15f
 	addi	r11,r11,8	/* use 32-bit syscall entries */
 	clrldi	r3,r3,32
 	clrldi	r4,r4,32
 	clrldi	r5,r5,32
 	clrldi	r6,r6,32
 	clrldi	r7,r7,32
 	clrldi	r8,r8,32
 15:
 	slwi	r0,r0,4
 	ldx	r10,r11,r0	/* Fetch system call handler [ptr] */
 	mtctr   r10
 	bctrl			/* Call handler */
 syscall_exit:
 	std	r3,RESULT(r1)
 #ifdef SHOW_SYSCALLS
 	bl	.do_show_syscall_exit
 	ld	r3,RESULT(r1)
 #endif
 	clrrdi	r12,r1,THREAD_SHIFT
 	ld	r8,_MSR(r1)
 #ifdef CONFIG_PPC_BOOK3S
 	/* No MSR:RI on BookE */
 	andi.	r10,r8,MSR_RI
 	beq-	unrecov_restore
 #endif
 	/* Disable interrupts so current_thread_info()->flags can't change,
 	 * and so that we don't get interrupted after loading SRR0/1.
 	 */
 #ifdef CONFIG_PPC_BOOK3E
 	wrteei	0
 #else
 	mfmsr	r10
 	rldicl	r10,r10,48,1
 	rotldi	r10,r10,16
 	mtmsrd	r10,1
 #endif /* CONFIG_PPC_BOOK3E */
 	ld	r9,TI_FLAGS(r12)
 	li	r11,-_LAST_ERRNO
 	andi.	r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
 	bne-	syscall_exit_work
 	cmpld	r3,r11
 	ld	r5,_CCR(r1)
 	bge-	syscall_error
 syscall_error_cont:
 	ld	r7,_NIP(r1)
 	stdcx.	r0,0,r1			/* to clear the reservation */
 	andi.	r6,r8,MSR_PR
 	ld	r4,_LINK(r1)
 	/*
 	 * Clear RI before restoring r13.  If we are returning to
 	 * userspace and we take an exception after restoring r13,
 	 * we end up corrupting the userspace r13 value.
 	 */
 #ifdef CONFIG_PPC_BOOK3S
 	/* No MSR:RI on BookE */
 	li	r12,MSR_RI
 	andc	r11,r10,r12
 	mtmsrd	r11,1			/* clear MSR.RI */
 #endif /* CONFIG_PPC_BOOK3S */
 	beq-	1f
 	ACCOUNT_CPU_USER_EXIT(r11, r12)
 	ld	r13,GPR13(r1)	/* only restore r13 if returning to usermode */
 1:	ld	r2,GPR2(r1)
 	ld	r1,GPR1(r1)
 	mtlr	r4
 	mtcr	r5
 	mtspr	SPRN_SRR0,r7
 	mtspr	SPRN_SRR1,r8
 	RFI
 	b	.	/* prevent speculative execution */
 syscall_error:
 	oris	r5,r5,0x1000	/* Set SO bit in CR */
 	neg	r3,r3
 	std	r5,_CCR(r1)
 	b	syscall_error_cont
 /* Traced system call support */
 syscall_dotrace:
 	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.do_syscall_trace_enter
 	/*
 	 * Restore argument registers possibly just changed.
 	 * We use the return value of do_syscall_trace_enter
 	 * for the call number to look up in the table (r0).
 	 */
 	mr	r0,r3
 	ld	r3,GPR3(r1)
 	ld	r4,GPR4(r1)
 	ld	r5,GPR5(r1)
 	ld	r6,GPR6(r1)
 	ld	r7,GPR7(r1)
 	ld	r8,GPR8(r1)
 	addi	r9,r1,STACK_FRAME_OVERHEAD
 	clrrdi	r10,r1,THREAD_SHIFT
 	ld	r10,TI_FLAGS(r10)
 	b	syscall_dotrace_cont
 syscall_enosys:
 	li	r3,-ENOSYS
 	b	syscall_exit
 syscall_exit_work:
 	/* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
 	 If TIF_NOERROR is set, just save r3 as it is. */
 	andi.	r0,r9,_TIF_RESTOREALL
 	beq+	0f
 	REST_NVGPRS(r1)
 	b	2f
 0:	cmpld	r3,r11		/* r10 is -LAST_ERRNO */
 	blt+	1f
 	andi.	r0,r9,_TIF_NOERROR
 	bne-	1f
 	ld	r5,_CCR(r1)
 	neg	r3,r3
 	oris	r5,r5,0x1000	/* Set SO bit in CR */
 	std	r5,_CCR(r1)
 1:	std	r3,GPR3(r1)
 2:	andi.	r0,r9,(_TIF_PERSYSCALL_MASK)
 	beq	4f
 	/* Clear per-syscall TIF flags if any are set.  */
 	li	r11,_TIF_PERSYSCALL_MASK
 	addi	r12,r12,TI_FLAGS
 3:	ldarx	r10,0,r12
 	andc	r10,r10,r11
 	stdcx.	r10,0,r12
 	bne-	3b
 	subi	r12,r12,TI_FLAGS
 4:	/* Anything else left to do? */
 	andi.	r0,r9,(_TIF_SYSCALL_T_OR_A|_TIF_SINGLESTEP)
 	beq	.ret_from_except_lite
 	/* Re-enable interrupts */
 #ifdef CONFIG_PPC_BOOK3E
 	wrteei	1
 #else
 	mfmsr	r10
 	ori	r10,r10,MSR_EE
 	mtmsrd	r10,1
 #endif /* CONFIG_PPC_BOOK3E */
 	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.do_syscall_trace_leave
 	b	.ret_from_except
 /* Save non-volatile GPRs, if not already saved. */
 _GLOBAL(save_nvgprs)
 	ld	r11,_TRAP(r1)
 	andi.	r0,r11,1
 	beqlr-
 	SAVE_NVGPRS(r1)
 	clrrdi	r0,r11,1
 	std	r0,_TRAP(r1)
 	blr
 /*
  * The sigsuspend and rt_sigsuspend system calls can call do_signal
  * and thus put the process into the stopped state where we might
  * want to examine its user state with ptrace.  Therefore we need
  * to save all the nonvolatile registers (r14 - r31) before calling
  * the C code.  Similarly, fork, vfork and clone need the full
  * register state on the stack so that it can be copied to the child.
  */
 _GLOBAL(ppc_fork)
 	bl	.save_nvgprs
 	bl	.sys_fork
 	b	syscall_exit
 _GLOBAL(ppc_vfork)
 	bl	.save_nvgprs
 	bl	.sys_vfork
 	b	syscall_exit
 _GLOBAL(ppc_clone)
 	bl	.save_nvgprs
 	bl	.sys_clone
 	b	syscall_exit
 _GLOBAL(ppc32_swapcontext)
 	bl	.save_nvgprs
 	bl	.compat_sys_swapcontext
 	b	syscall_exit
 _GLOBAL(ppc64_swapcontext)
 	bl	.save_nvgprs
 	bl	.sys_swapcontext
 	b	syscall_exit
 _GLOBAL(ret_from_fork)
 	bl	.schedule_tail
 	REST_NVGPRS(r1)
 	li	r3,0
 	b	syscall_exit
 /*
  * This routine switches between two different tasks.  The process
  * state of one is saved on its kernel stack.  Then the state
  * of the other is restored from its kernel stack.  The memory
  * management hardware is updated to the second process's state.
  * Finally, we can return to the second process, via ret_from_except.
  * On entry, r3 points to the THREAD for the current task, r4
  * points to the THREAD for the new task.
  *
  * Note: there are two ways to get to the "going out" portion
  * of this code; either by coming in via the entry (_switch)
  * or via "fork" which must set up an environment equivalent
  * to the "_switch" path.  If you change this you'll have to change
  * the fork code also.
  *
  * The code which creates the new task context is in 'copy_thread'
  * in arch/powerpc/kernel/process.c
  */
 	.align	7
 _GLOBAL(_switch)
 	mflr	r0
 	std	r0,16(r1)
 	stdu	r1,-SWITCH_FRAME_SIZE(r1)
 	/* r3-r13 are caller saved -- Cort */
 	SAVE_8GPRS(14, r1)
 	SAVE_10GPRS(22, r1)
 	mflr	r20		/* Return to switch caller */
 	mfmsr	r22
 	li	r0, MSR_FP
 #ifdef CONFIG_VSX
 BEGIN_FTR_SECTION
 	oris	r0,r0,MSR_VSX@h	/* Disable VSX */
 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
 #endif /* CONFIG_VSX */
 #ifdef CONFIG_ALTIVEC
 BEGIN_FTR_SECTION
 	oris	r0,r0,MSR_VEC@h	/* Disable altivec */
 	mfspr	r24,SPRN_VRSAVE	/* save vrsave register value */
 	std	r24,THREAD_VRSAVE(r3)
 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 #endif /* CONFIG_ALTIVEC */
 	and.	r0,r0,r22
 	beq+	1f
 	andc	r22,r22,r0
 	MTMSRD(r22)
 	isync
 1:	std	r20,_NIP(r1)
 	mfcr	r23
 	std	r23,_CCR(r1)
 	std	r1,KSP(r3)	/* Set old stack pointer */
 #ifdef CONFIG_SMP
 	/* We need a sync somewhere here to make sure that if the
 	 * previous task gets rescheduled on another CPU, it sees all
 	 * stores it has performed on this one.
 	 */
 	sync
 #endif /* CONFIG_SMP */
 	addi	r6,r4,-THREAD	/* Convert THREAD to 'current' */
 	std	r6,PACACURRENT(r13)	/* Set new 'current' */
 	ld	r8,KSP(r4)	/* new stack pointer */
 #ifdef CONFIG_PPC_BOOK3S
 BEGIN_FTR_SECTION
   BEGIN_FTR_SECTION_NESTED(95)
 	clrrdi	r6,r8,28	/* get its ESID */
 	clrrdi	r9,r1,28	/* get current sp ESID */
   FTR_SECTION_ELSE_NESTED(95)
 	clrrdi	r6,r8,40	/* get its 1T ESID */
 	clrrdi	r9,r1,40	/* get current sp 1T ESID */
   ALT_FTR_SECTION_END_NESTED_IFCLR(CPU_FTR_1T_SEGMENT, 95)
 FTR_SECTION_ELSE
 	b	2f
 ALT_FTR_SECTION_END_IFSET(CPU_FTR_SLB)
 	clrldi.	r0,r6,2		/* is new ESID c00000000? */
 	cmpd	cr1,r6,r9	/* or is new ESID the same as current ESID? */
 	cror	eq,4*cr1+eq,eq
 	beq	2f		/* if yes, don't slbie it */
 	/* Bolt in the new stack SLB entry */
 	ld	r7,KSP_VSID(r4)	/* Get new stack's VSID */
 	oris	r0,r6,(SLB_ESID_V)@h
 	ori	r0,r0,(SLB_NUM_BOLTED-1)@l
 BEGIN_FTR_SECTION
 	li	r9,MMU_SEGSIZE_1T	/* insert B field */
 	oris	r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
 	rldimi	r7,r9,SLB_VSID_SSIZE_SHIFT,0
 END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT)
 	/* Update the last bolted SLB.  No write barriers are needed
 	 * here, provided we only update the current CPU's SLB shadow
 	 * buffer.
 	 */
 	ld	r9,PACA_SLBSHADOWPTR(r13)
 	li	r12,0
 	std	r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
 	std	r7,SLBSHADOW_STACKVSID(r9)  /* Save VSID */
 	std	r0,SLBSHADOW_STACKESID(r9)  /* Save ESID */
 	/* No need to check for CPU_FTR_NO_SLBIE_B here, since when
 	 * we have 1TB segments, the only CPUs known to have the errata
 	 * only support less than 1TB of system memory and we'll never
 	 * actually hit this code path.
 	 */
 	slbie	r6
 	slbie	r6		/* Workaround POWER5 < DD2.1 issue */
 	slbmte	r7,r0
 	isync
 2:
 #endif /* !CONFIG_PPC_BOOK3S */
 	clrrdi	r7,r8,THREAD_SHIFT	/* base of new stack */
 	/* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
 	   because we don't need to leave the 288-byte ABI gap at the
 	   top of the kernel stack. */
 	addi	r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
 	mr	r1,r8		/* start using new stack pointer */
 	std	r7,PACAKSAVE(r13)
 	ld	r6,_CCR(r1)
 	mtcrf	0xFF,r6
 #ifdef CONFIG_ALTIVEC
 BEGIN_FTR_SECTION
 	ld	r0,THREAD_VRSAVE(r4)
 	mtspr	SPRN_VRSAVE,r0		/* if G4, restore VRSAVE reg */
 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
 #endif /* CONFIG_ALTIVEC */
 	/* r3-r13 are destroyed -- Cort */
 	REST_8GPRS(14, r1)
 	REST_10GPRS(22, r1)
 	/* convert old thread to its task_struct for return value */
 	addi	r3,r3,-THREAD
 	ld	r7,_NIP(r1)	/* Return to _switch caller in new task */
 	mtlr	r7
 	addi	r1,r1,SWITCH_FRAME_SIZE
 	blr
 	.align	7
 _GLOBAL(ret_from_except)
 	ld	r11,_TRAP(r1)
 	andi.	r0,r11,1
 	bne	.ret_from_except_lite
 	REST_NVGPRS(r1)
 _GLOBAL(ret_from_except_lite)
 	/*
 	 * Disable interrupts so that current_thread_info()->flags
 	 * can't change between when we test it and when we return
 	 * from the interrupt.
 	 */
 #ifdef CONFIG_PPC_BOOK3E
 	wrteei	0
 #else
 	mfmsr	r10		/* Get current interrupt state */
 	rldicl	r9,r10,48,1	/* clear MSR_EE */
 	rotldi	r9,r9,16
 	mtmsrd	r9,1		/* Update machine state */
 #endif /* CONFIG_PPC_BOOK3E */
 #ifdef CONFIG_PREEMPT
 	clrrdi	r9,r1,THREAD_SHIFT	/* current_thread_info() */
 	li	r0,_TIF_NEED_RESCHED	/* bits to check */
 	ld	r3,_MSR(r1)
 	ld	r4,TI_FLAGS(r9)
 	/* Move MSR_PR bit in r3 to _TIF_SIGPENDING position in r0 */
 	rlwimi	r0,r3,32+TIF_SIGPENDING-MSR_PR_LG,_TIF_SIGPENDING
 	and.	r0,r4,r0	/* check NEED_RESCHED and maybe SIGPENDING */
 	bne	do_work
 #else /* !CONFIG_PREEMPT */
 	ld	r3,_MSR(r1)	/* Returning to user mode? */
 	andi.	r3,r3,MSR_PR
 	beq	restore		/* if not, just restore regs and return */
 	/* Check current_thread_info()->flags */
 	clrrdi	r9,r1,THREAD_SHIFT
 	ld	r4,TI_FLAGS(r9)
 	andi.	r0,r4,_TIF_USER_WORK_MASK
 	bne	do_work
 #endif
 restore:
 BEGIN_FW_FTR_SECTION
 	ld	r5,SOFTE(r1)
 FW_FTR_SECTION_ELSE
 	b	.Liseries_check_pending_irqs
 ALT_FW_FTR_SECTION_END_IFCLR(FW_FEATURE_ISERIES)
 2:
 	TRACE_AND_RESTORE_IRQ(r5);
-#ifdef CONFIG_PERF_EVENTS
-	/* check paca->perf_event_pending if we're enabling ints */
-	lbz	r3,PACAPERFPEND(r13)
-	and.	r3,r3,r5
-	beq	27f
-	bl	.perf_event_do_pending
-27:
-#endif /* CONFIG_PERF_EVENTS */
 	/* extract EE bit and use it to restore paca->hard_enabled */
 	ld	r3,_MSR(r1)
 	rldicl	r4,r3,49,63		/* r0 = (r3 >> 15) & 1 */
 	stb	r4,PACAHARDIRQEN(r13)
 #ifdef CONFIG_PPC_BOOK3E
 	b	.exception_return_book3e
 #else
 	ld	r4,_CTR(r1)
 	ld	r0,_LINK(r1)
 	mtctr	r4
 	mtlr	r0
 	ld	r4,_XER(r1)
 	mtspr	SPRN_XER,r4
 	REST_8GPRS(5, r1)
 	andi.	r0,r3,MSR_RI
 	beq-	unrecov_restore
 	stdcx.	r0,0,r1		/* to clear the reservation */
 	/*
 	 * Clear RI before restoring r13.  If we are returning to
 	 * userspace and we take an exception after restoring r13,
 	 * we end up corrupting the userspace r13 value.
 	 */
 	mfmsr	r4
 	andc	r4,r4,r0	/* r0 contains MSR_RI here */
 	mtmsrd	r4,1
 	/*
 	 * r13 is our per cpu area, only restore it if we are returning to
 	 * userspace
 	 */
 	andi.	r0,r3,MSR_PR
 	beq	1f
 	ACCOUNT_CPU_USER_EXIT(r2, r4)
 	REST_GPR(13, r1)
 1:
 	mtspr	SPRN_SRR1,r3
 	ld	r2,_CCR(r1)
 	mtcrf	0xFF,r2
 	ld	r2,_NIP(r1)
 	mtspr	SPRN_SRR0,r2
 	ld	r0,GPR0(r1)
 	ld	r2,GPR2(r1)
 	ld	r3,GPR3(r1)
 	ld	r4,GPR4(r1)
 	ld	r1,GPR1(r1)
 	rfid
 	b	.	/* prevent speculative execution */
 #endif /* CONFIG_PPC_BOOK3E */
 .Liseries_check_pending_irqs:
 #ifdef CONFIG_PPC_ISERIES
 	ld	r5,SOFTE(r1)
 	cmpdi	0,r5,0
 	beq	2b
 	/* Check for pending interrupts (iSeries) */
 	ld	r3,PACALPPACAPTR(r13)
 	ld	r3,LPPACAANYINT(r3)
 	cmpdi	r3,0
 	beq+	2b			/* skip do_IRQ if no interrupts */
 	li	r3,0
 	stb	r3,PACASOFTIRQEN(r13)	/* ensure we are soft-disabled */
 #ifdef CONFIG_TRACE_IRQFLAGS
 	bl	.trace_hardirqs_off
 	mfmsr	r10
 #endif
 	ori	r10,r10,MSR_EE
 	mtmsrd	r10			/* hard-enable again */
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.do_IRQ
 	b	.ret_from_except_lite		/* loop back and handle more */
 #endif
 do_work:
 #ifdef CONFIG_PREEMPT
 	andi.	r0,r3,MSR_PR	/* Returning to user mode? */
 	bne	user_work
 	/* Check that preempt_count() == 0 and interrupts are enabled */
 	lwz	r8,TI_PREEMPT(r9)
 	cmpwi	cr1,r8,0
 	ld	r0,SOFTE(r1)
 	cmpdi	r0,0
 	crandc	eq,cr1*4+eq,eq
 	bne	restore
 	/* Here we are preempting the current task.
 	 *
 	 * Ensure interrupts are soft-disabled. We also properly mark
 	 * the PACA to reflect the fact that they are hard-disabled
 	 * and trace the change
 	 */
 	li	r0,0
 	stb	r0,PACASOFTIRQEN(r13)
 	stb	r0,PACAHARDIRQEN(r13)
 	TRACE_DISABLE_INTS
 	/* Call the scheduler with soft IRQs off */
 1:	bl	.preempt_schedule_irq
 	/* Hard-disable interrupts again (and update PACA) */
 #ifdef CONFIG_PPC_BOOK3E
 	wrteei	0
 #else
 	mfmsr	r10
 	rldicl	r10,r10,48,1
 	rotldi	r10,r10,16
 	mtmsrd	r10,1
 #endif /* CONFIG_PPC_BOOK3E */
 	li	r0,0
 	stb	r0,PACAHARDIRQEN(r13)
 	/* Re-test flags and eventually loop */
 	clrrdi	r9,r1,THREAD_SHIFT
 	ld	r4,TI_FLAGS(r9)
 	andi.	r0,r4,_TIF_NEED_RESCHED
 	bne	1b
 	b	restore
 user_work:
 #endif /* CONFIG_PREEMPT */
 	/* Enable interrupts */
 #ifdef CONFIG_PPC_BOOK3E
 	wrteei	1
 #else
 	ori	r10,r10,MSR_EE
 	mtmsrd	r10,1
 #endif /* CONFIG_PPC_BOOK3E */
 	andi.	r0,r4,_TIF_NEED_RESCHED
 	beq	1f
 	bl	.schedule
 	b	.ret_from_except_lite
 1:	bl	.save_nvgprs
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.do_signal
 	b	.ret_from_except
 unrecov_restore:
 	addi	r3,r1,STACK_FRAME_OVERHEAD
 	bl	.unrecoverable_exception
 	b	unrecov_restore
 #ifdef CONFIG_PPC_RTAS
 /*
  * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
  * called with the MMU off.
  *
  * In addition, we need to be in 32b mode, at least for now.
  *
  * Note: r3 is an input parameter to rtas, so don't trash it...
  */
 _GLOBAL(enter_rtas)
 	mflr	r0
 	std	r0,16(r1)
         stdu	r1,-RTAS_FRAME_SIZE(r1)	/* Save SP and create stack space. */
 	/* Because RTAS is running in 32b mode, it clobbers the high order half
 	 * of all registers that it saves.  We therefore save those registers
 	 * RTAS might touch to the stack.  (r0, r3-r13 are caller saved)
    	 */
 	SAVE_GPR(2, r1)			/* Save the TOC */
 	SAVE_GPR(13, r1)		/* Save paca */
 	SAVE_8GPRS(14, r1)		/* Save the non-volatiles */
 	SAVE_10GPRS(22, r1)		/* ditto */
 	mfcr	r4
 	std	r4,_CCR(r1)
 	mfctr	r5
 	std	r5,_CTR(r1)
 	mfspr	r6,SPRN_XER
 	std	r6,_XER(r1)
 	mfdar	r7
 	std	r7,_DAR(r1)
 	mfdsisr	r8
 	std	r8,_DSISR(r1)
 	/* Temporary workaround to clear CR until RTAS can be modified to
 	 * ignore all bits.
 	 */
 	li	r0,0
 	mtcr	r0
 #ifdef CONFIG_BUG
 	/* There is no way it is acceptable to get here with interrupts enabled,
 	 * check it with the asm equivalent of WARN_ON
 	 */
 	lbz	r0,PACASOFTIRQEN(r13)
 1:	tdnei	r0,0
 	EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
 #endif
 	/* Hard-disable interrupts */
 	mfmsr	r6
 	rldicl	r7,r6,48,1
 	rotldi	r7,r7,16
 	mtmsrd	r7,1
 	/* Unfortunately, the stack pointer and the MSR are also clobbered,
 	 * so they are saved in the PACA which allows us to restore
 	 * our original state after RTAS returns.
          */
 	std	r1,PACAR1(r13)
         std	r6,PACASAVEDMSR(r13)
 	/* Setup our real return addr */
 	LOAD_REG_ADDR(r4,.rtas_return_loc)
 	clrldi	r4,r4,2			/* convert to realmode address */
        	mtlr	r4
 	li	r0,0
 	ori	r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
 	andc	r0,r6,r0
         li      r9,1
         rldicr  r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
 	ori	r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI
 	andc	r6,r0,r9
 	sync				/* disable interrupts so SRR0/1 */
 	mtmsrd	r0			/* don't get trashed */
 	LOAD_REG_ADDR(r4, rtas)
 	ld	r5,RTASENTRY(r4)	/* get the rtas->entry value */
 	ld	r4,RTASBASE(r4)		/* get the rtas->base value */
 	mtspr	SPRN_SRR0,r5
 	mtspr	SPRN_SRR1,r6
 	rfid
 	b	.	/* prevent speculative execution */
 _STATIC(rtas_return_loc)
 	/* relocation is off at this point */
 	mfspr	r4,SPRN_SPRG_PACA	/* Get PACA */
 	clrldi	r4,r4,2			/* convert to realmode address */
 	bcl	20,31,$+4
 0:	mflr	r3
 	ld	r3,(1f-0b)(r3)		/* get &.rtas_restore_regs */
 	mfmsr   r6
 	li	r0,MSR_RI
 	andc	r6,r6,r0
 	sync
 	mtmsrd  r6
         ld	r1,PACAR1(r4)           /* Restore our SP */
         ld	r4,PACASAVEDMSR(r4)     /* Restore our MSR */
 	mtspr	SPRN_SRR0,r3
 	mtspr	SPRN_SRR1,r4
 	rfid
 	b	.	/* prevent speculative execution */
 	.align	3
 1:	.llong	.rtas_restore_regs
 _STATIC(rtas_restore_regs)
 	/* relocation is on at this point */
 	REST_GPR(2, r1)			/* Restore the TOC */
 	REST_GPR(13, r1)		/* Restore paca */
 	REST_8GPRS(14, r1)		/* Restore the non-volatiles */
 	REST_10GPRS(22, r1)		/* ditto */
 	mfspr	r13,SPRN_SPRG_PACA
 	ld	r4,_CCR(r1)
 	mtcr	r4
 	ld	r5,_CTR(r1)
 	mtctr	r5
 	ld	r6,_XER(r1)
 	mtspr	SPRN_XER,r6
 	ld	r7,_DAR(r1)
 	mtdar	r7
 	ld	r8,_DSISR(r1)
 	mtdsisr	r8
         addi	r1,r1,RTAS_FRAME_SIZE	/* Unstack our frame */
 	ld	r0,16(r1)		/* get return address */
 	mtlr    r0
         blr				/* return to caller */
 #endif /* CONFIG_PPC_RTAS */
 _GLOBAL(enter_prom)
 	mflr	r0
 	std	r0,16(r1)
         stdu	r1,-PROM_FRAME_SIZE(r1)	/* Save SP and create stack space */
 	/* Because PROM is running in 32b mode, it clobbers the high order half
 	 * of all registers that it saves.  We therefore save those registers
 	 * PROM might touch to the stack.  (r0, r3-r13 are caller saved)
    	 */
 	SAVE_GPR(2, r1)
 	SAVE_GPR(13, r1)
 	SAVE_8GPRS(14, r1)
 	SAVE_10GPRS(22, r1)
 	mfcr	r10
 	mfmsr	r11
 	std	r10,_CCR(r1)
 	std	r11,_MSR(r1)
 	/* Get the PROM entrypoint */
 	mtlr	r4
 	/* Switch MSR to 32 bits mode
 	 */
 #ifdef CONFIG_PPC_BOOK3E
 	rlwinm	r11,r11,0,1,31
 	mtmsr	r11
 #else /* CONFIG_PPC_BOOK3E */
         mfmsr   r11
         li      r12,1
         rldicr  r12,r12,MSR_SF_LG,(63-MSR_SF_LG)
         andc    r11,r11,r12
         li      r12,1
         rldicr  r12,r12,MSR_ISF_LG,(63-MSR_ISF_LG)
         andc    r11,r11,r12
         mtmsrd  r11
 #endif /* CONFIG_PPC_BOOK3E */
         isync
 	/* Enter PROM here... */
 	blrl
 	/* Just make sure that r1 top 32 bits didn't get
 	 * corrupt by OF
 	 */
 	rldicl	r1,r1,0,32
 	/* Restore the MSR (back to 64 bits) */
 	ld	r0,_MSR(r1)
 	MTMSRD(r0)
         isync
 	/* Restore other registers */
 	REST_GPR(2, r1)
 	REST_GPR(13, r1)
 	REST_8GPRS(14, r1)
 	REST_10GPRS(22, r1)
 	ld	r4,_CCR(r1)
 	mtcr	r4
         addi	r1,r1,PROM_FRAME_SIZE
 	ld	r0,16(r1)
 	mtlr    r0
         blr
 #ifdef CONFIG_FUNCTION_TRACER
 #ifdef CONFIG_DYNAMIC_FTRACE
 _GLOBAL(mcount)
 _GLOBAL(_mcount)
 	blr
 _GLOBAL(ftrace_caller)
 	/* Taken from output of objdump from lib64/glibc */
 	mflr	r3
 	ld	r11, 0(r1)
 	stdu	r1, -112(r1)
 	std	r3, 128(r1)
 	ld	r4, 16(r11)
 	subi	r3, r3, MCOUNT_INSN_SIZE
 .globl ftrace_call
 ftrace_call:
 	bl	ftrace_stub
 	nop
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 .globl ftrace_graph_call
 ftrace_graph_call:
 	b	ftrace_graph_stub
 _GLOBAL(ftrace_graph_stub)
 #endif
 	ld	r0, 128(r1)
 	mtlr	r0
 	addi	r1, r1, 112
 _GLOBAL(ftrace_stub)
 	blr
 #else
 _GLOBAL(mcount)
 	blr
 _GLOBAL(_mcount)
 	/* Taken from output of objdump from lib64/glibc */
 	mflr	r3
 	ld	r11, 0(r1)
 	stdu	r1, -112(r1)
 	std	r3, 128(r1)
 	ld	r4, 16(r11)
 	subi	r3, r3, MCOUNT_INSN_SIZE
 	LOAD_REG_ADDR(r5,ftrace_trace_function)
 	ld	r5,0(r5)
 	ld	r5,0(r5)
 	mtctr	r5
 	bctrl
 	nop
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 	b	ftrace_graph_caller
 #endif
 	ld	r0, 128(r1)
 	mtlr	r0
 	addi	r1, r1, 112
 _GLOBAL(ftrace_stub)
 	blr
 #endif /* CONFIG_DYNAMIC_FTRACE */
 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 _GLOBAL(ftrace_graph_caller)
 	/* load r4 with local address */
 	ld	r4, 128(r1)
 	subi	r4, r4, MCOUNT_INSN_SIZE
 	/* get the parent address */
 	ld	r11, 112(r1)
 	addi	r3, r11, 16
 	bl	.prepare_ftrace_return
 	nop
 	ld	r0, 128(r1)
 	mtlr	r0
 	addi	r1, r1, 112
 	blr
 _GLOBAL(return_to_handler)
 	/* need to save return values */
 	std	r4,  -24(r1)
 	std	r3,  -16(r1)
 	std	r31, -8(r1)
 	mr	r31, r1
 	stdu	r1, -112(r1)
 	bl	.ftrace_return_to_handler
 	nop
 	/* return value has real return address */
 	mtlr	r3
 	ld	r1, 0(r1)
 	ld	r4,  -24(r1)
 	ld	r3,  -16(r1)
 	ld	r31, -8(r1)
 	/* Jump back to real return address */
 	blr
 _GLOBAL(mod_return_to_handler)
 	/* need to save return values */
 	std	r4,  -32(r1)
 	std	r3,  -24(r1)
 	/* save TOC */
 	std	r2,  -16(r1)
 	std	r31, -8(r1)
 	mr	r31, r1
 	stdu	r1, -112(r1)
 	/*
 	 * We are in a module using the module's TOC.
 	 * Switch to our TOC to run inside the core kernel.
 	 */
 	ld	r2, PACATOC(r13)
 	bl	.ftrace_return_to_handler
 	nop
 	/* return value has real return address */
 	mtlr	r3
 	ld	r1, 0(r1)
 	ld	r4,  -32(r1)
 	ld	r3,  -24(r1)
 	ld	r2,  -16(r1)
 	ld	r31, -8(r1)
 	/* Jump back to real return address */
 	blr
 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 #endif /* CONFIG_FUNCTION_TRACER */

arch/powerpc/kernel/irq.c

Diff comments View file @ be83567

 /*
  *  Derived from arch/i386/kernel/irq.c
  *    Copyright (C) 1992 Linus Torvalds
  *  Adapted from arch/i386 by Gary Thomas
  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  *  Updated and modified by Cort Dougan <cort@fsmlabs.com>
  *    Copyright (C) 1996-2001 Cort Dougan
  *  Adapted for Power Macintosh by Paul Mackerras
  *    Copyright (C) 1996 Paul Mackerras (paulus@cs.anu.edu.au)
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License
  * as published by the Free Software Foundation; either version
  * 2 of the License, or (at your option) any later version.
  *
  * This file contains the code used by various IRQ handling routines:
  * asking for different IRQ's should be done through these routines
  * instead of just grabbing them. Thus setups with different IRQ numbers
  * shouldn't result in any weird surprises, and installing new handlers
  * should be easier.
  *
  * The MPC8xx has an interrupt mask in the SIU.  If a bit is set, the
  * interrupt is _enabled_.  As expected, IRQ0 is bit 0 in the 32-bit
  * mask register (of which only 16 are defined), hence the weird shifting
  * and complement of the cached_irq_mask.  I want to be able to stuff
  * this right into the SIU SMASK register.
  * Many of the prep/chrp functions are conditional compiled on CONFIG_8xx
  * to reduce code space and undefined function references.
  */
 #undef DEBUG
 #include <linux/module.h>
 #include <linux/threads.h>
 #include <linux/kernel_stat.h>
 #include <linux/signal.h>
 #include <linux/sched.h>
 #include <linux/ptrace.h>
 #include <linux/ioport.h>
 #include <linux/interrupt.h>
 #include <linux/timex.h>
 #include <linux/init.h>
 #include <linux/slab.h>
 #include <linux/delay.h>
 #include <linux/irq.h>
 #include <linux/seq_file.h>
 #include <linux/cpumask.h>
 #include <linux/profile.h>
 #include <linux/bitops.h>
 #include <linux/list.h>
 #include <linux/radix-tree.h>
 #include <linux/mutex.h>
 #include <linux/bootmem.h>
 #include <linux/pci.h>
 #include <linux/debugfs.h>
-#include <linux/perf_event.h>
 #include <asm/uaccess.h>
 #include <asm/system.h>
 #include <asm/io.h>
 #include <asm/pgtable.h>
 #include <asm/irq.h>
 #include <asm/cache.h>
 #include <asm/prom.h>
 #include <asm/ptrace.h>
 #include <asm/machdep.h>
 #include <asm/udbg.h>
 #ifdef CONFIG_PPC64
 #include <asm/paca.h>
 #include <asm/firmware.h>
 #include <asm/lv1call.h>
 #endif
 #define CREATE_TRACE_POINTS
 #include <asm/trace.h>
 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
 EXPORT_PER_CPU_SYMBOL(irq_stat);
 int __irq_offset_value;
 #ifdef CONFIG_PPC32
 EXPORT_SYMBOL(__irq_offset_value);
 atomic_t ppc_n_lost_interrupts;
 #ifdef CONFIG_TAU_INT
 extern int tau_initialized;
 extern int tau_interrupts(int);
 #endif
 #endif /* CONFIG_PPC32 */
 #ifdef CONFIG_PPC64
 #ifndef CONFIG_SPARSE_IRQ
 EXPORT_SYMBOL(irq_desc);
 #endif
 int distribute_irqs = 1;
 static inline notrace unsigned long get_hard_enabled(void)
 {
 	unsigned long enabled;
 	__asm__ __volatile__("lbz %0,%1(13)"
 	: "=r" (enabled) : "i" (offsetof(struct paca_struct, hard_enabled)));
 	return enabled;
 }
 static inline notrace void set_soft_enabled(unsigned long enable)
 {
 	__asm__ __volatile__("stb %0,%1(13)"
 	: : "r" (enable), "i" (offsetof(struct paca_struct, soft_enabled)));
 }
 notrace void raw_local_irq_restore(unsigned long en)
 {
 	/*
 	 * get_paca()->soft_enabled = en;
 	 * Is it ever valid to use local_irq_restore(0) when soft_enabled is 1?
 	 * That was allowed before, and in such a case we do need to take care
 	 * that gcc will set soft_enabled directly via r13, not choose to use
 	 * an intermediate register, lest we're preempted to a different cpu.
 	 */
 	set_soft_enabled(en);
 	if (!en)
 		return;
 #ifdef CONFIG_PPC_STD_MMU_64
 	if (firmware_has_feature(FW_FEATURE_ISERIES)) {
 		/*
 		 * Do we need to disable preemption here?  Not really: in the
 		 * unlikely event that we're preempted to a different cpu in
 		 * between getting r13, loading its lppaca_ptr, and loading
 		 * its any_int, we might call iseries_handle_interrupts without
 		 * an interrupt pending on the new cpu, but that's no disaster,
 		 * is it?  And the business of preempting us off the old cpu
 		 * would itself involve a local_irq_restore which handles the
 		 * interrupt to that cpu.
 		 *
 		 * But use "local_paca->lppaca_ptr" instead of "get_lppaca()"
 		 * to avoid any preemption checking added into get_paca().
 		 */
 		if (local_paca->lppaca_ptr->int_dword.any_int)
 			iseries_handle_interrupts();
 	}
 #endif /* CONFIG_PPC_STD_MMU_64 */
-	if (test_perf_event_pending()) {
-		clear_perf_event_pending();
-		perf_event_do_pending();
-	}
 	/*
 	 * if (get_paca()->hard_enabled) return;
 	 * But again we need to take care that gcc gets hard_enabled directly
 	 * via r13, not choose to use an intermediate register, lest we're
 	 * preempted to a different cpu in between the two instructions.
 	 */
 	if (get_hard_enabled())
 		return;
 	/*
 	 * Need to hard-enable interrupts here.  Since currently disabled,
 	 * no need to take further asm precautions against preemption; but
 	 * use local_paca instead of get_paca() to avoid preemption checking.
 	 */
 	local_paca->hard_enabled = en;
 	if ((int)mfspr(SPRN_DEC) < 0)
 		mtspr(SPRN_DEC, 1);
 	/*
 	 * Force the delivery of pending soft-disabled interrupts on PS3.
 	 * Any HV call will have this side effect.
 	 */
 	if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
 		u64 tmp;
 		lv1_get_version_info(&tmp);
 	}
 	__hard_irq_enable();
 }
 EXPORT_SYMBOL(raw_local_irq_restore);
 #endif /* CONFIG_PPC64 */
 static int show_other_interrupts(struct seq_file *p, int prec)
 {
 	int j;
 #if defined(CONFIG_PPC32) && defined(CONFIG_TAU_INT)
 	if (tau_initialized) {
 		seq_printf(p, "%*s: ", prec, "TAU");
 		for_each_online_cpu(j)
 			seq_printf(p, "%10u ", tau_interrupts(j));
 		seq_puts(p, "  PowerPC             Thermal Assist (cpu temp)\n");
 	}
 #endif /* CONFIG_PPC32 && CONFIG_TAU_INT */
 	seq_printf(p, "%*s: ", prec, "LOC");
 	for_each_online_cpu(j)
 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs);
         seq_printf(p, "  Local timer interrupts\n");
 	seq_printf(p, "%*s: ", prec, "SPU");
 	for_each_online_cpu(j)
 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).spurious_irqs);
 	seq_printf(p, "  Spurious interrupts\n");
 	seq_printf(p, "%*s: ", prec, "CNT");
 	for_each_online_cpu(j)
 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).pmu_irqs);
 	seq_printf(p, "  Performance monitoring interrupts\n");
 	seq_printf(p, "%*s: ", prec, "MCE");
 	for_each_online_cpu(j)
 		seq_printf(p, "%10u ", per_cpu(irq_stat, j).mce_exceptions);
 	seq_printf(p, "  Machine check exceptions\n");
 	return 0;
 }
 int show_interrupts(struct seq_file *p, void *v)
 {
 	unsigned long flags, any_count = 0;
 	int i = *(loff_t *) v, j, prec;
 	struct irqaction *action;
 	struct irq_desc *desc;
 	if (i > nr_irqs)
 		return 0;
 	for (prec = 3, j = 1000; prec < 10 && j <= nr_irqs; ++prec)
 		j *= 10;
 	if (i == nr_irqs)
 		return show_other_interrupts(p, prec);
 	/* print header */
 	if (i == 0) {
 		seq_printf(p, "%*s", prec + 8, "");
 		for_each_online_cpu(j)
 			seq_printf(p, "CPU%-8d", j);
 		seq_putc(p, '\n');
 	}
 	desc = irq_to_desc(i);
 	if (!desc)
 		return 0;
 	raw_spin_lock_irqsave(&desc->lock, flags);
 	for_each_online_cpu(j)
 		any_count |= kstat_irqs_cpu(i, j);
 	action = desc->action;
 	if (!action && !any_count)
 		goto out;
 	seq_printf(p, "%*d: ", prec, i);
 	for_each_online_cpu(j)
 		seq_printf(p, "%10u ", kstat_irqs_cpu(i, j));
 	if (desc->chip)
 		seq_printf(p, "  %-16s", desc->chip->name);
 	else
 		seq_printf(p, "  %-16s", "None");
 	seq_printf(p, " %-8s", (desc->status & IRQ_LEVEL) ? "Level" : "Edge");
 	if (action) {
 		seq_printf(p, "     %s", action->name);
 		while ((action = action->next) != NULL)
 			seq_printf(p, ", %s", action->name);
 	}
 	seq_putc(p, '\n');
 out:
 	raw_spin_unlock_irqrestore(&desc->lock, flags);
 	return 0;
 }
 /*
  * /proc/stat helpers
  */
 u64 arch_irq_stat_cpu(unsigned int cpu)
 {
 	u64 sum = per_cpu(irq_stat, cpu).timer_irqs;
 	sum += per_cpu(irq_stat, cpu).pmu_irqs;
 	sum += per_cpu(irq_stat, cpu).mce_exceptions;
 	sum += per_cpu(irq_stat, cpu).spurious_irqs;
 	return sum;
 }
 #ifdef CONFIG_HOTPLUG_CPU
 void fixup_irqs(cpumask_t map)
 {
 	struct irq_desc *desc;
 	unsigned int irq;
 	static int warned;
 	for_each_irq(irq) {
 		cpumask_t mask;
 		desc = irq_to_desc(irq);
 		if (desc && desc->status & IRQ_PER_CPU)
 			continue;
 		cpumask_and(&mask, desc->affinity, &map);
 		if (any_online_cpu(mask) == NR_CPUS) {
 			printk("Breaking affinity for irq %i\n", irq);
 			mask = map;
 		}
 		if (desc->chip->set_affinity)
 			desc->chip->set_affinity(irq, &mask);
 		else if (desc->action && !(warned++))
 			printk("Cannot set affinity for irq %i\n", irq);
 	}
 	local_irq_enable();
 	mdelay(1);
 	local_irq_disable();
 }
 #endif
 #ifdef CONFIG_IRQSTACKS
 static inline void handle_one_irq(unsigned int irq)
 {
 	struct thread_info *curtp, *irqtp;
 	unsigned long saved_sp_limit;
 	struct irq_desc *desc;
 	/* Switch to the irq stack to handle this */
 	curtp = current_thread_info();
 	irqtp = hardirq_ctx[smp_processor_id()];
 	if (curtp == irqtp) {
 		/* We're already on the irq stack, just handle it */
 		generic_handle_irq(irq);
 		return;
 	}
 	desc = irq_to_desc(irq);
 	saved_sp_limit = current->thread.ksp_limit;
 	irqtp->task = curtp->task;
 	irqtp->flags = 0;
 	/* Copy the softirq bits in preempt_count so that the
 	 * softirq checks work in the hardirq context. */
 	irqtp->preempt_count = (irqtp->preempt_count & ~SOFTIRQ_MASK) |
 			       (curtp->preempt_count & SOFTIRQ_MASK);
 	current->thread.ksp_limit = (unsigned long)irqtp +
 		_ALIGN_UP(sizeof(struct thread_info), 16);
 	call_handle_irq(irq, desc, irqtp, desc->handle_irq);
 	current->thread.ksp_limit = saved_sp_limit;
 	irqtp->task = NULL;
 	/* Set any flag that may have been set on the
 	 * alternate stack
 	 */
 	if (irqtp->flags)
 		set_bits(irqtp->flags, &curtp->flags);
 }
 #else
 static inline void handle_one_irq(unsigned int irq)
 {
 	generic_handle_irq(irq);
 }
 #endif
 static inline void check_stack_overflow(void)
 {
 #ifdef CONFIG_DEBUG_STACKOVERFLOW
 	long sp;
 	sp = __get_SP() & (THREAD_SIZE-1);
 	/* check for stack overflow: is there less than 2KB free? */
 	if (unlikely(sp < (sizeof(struct thread_info) + 2048))) {
 		printk("do_IRQ: stack overflow: %ld\n",
 			sp - sizeof(struct thread_info));
 		dump_stack();
 	}
 #endif
 }
 void do_IRQ(struct pt_regs *regs)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
 	unsigned int irq;
 	trace_irq_entry(regs);
 	irq_enter();
 	check_stack_overflow();
 	irq = ppc_md.get_irq();
 	if (irq != NO_IRQ && irq != NO_IRQ_IGNORE)
 		handle_one_irq(irq);
 	else if (irq != NO_IRQ_IGNORE)
 		__get_cpu_var(irq_stat).spurious_irqs++;
 	irq_exit();
 	set_irq_regs(old_regs);
 #ifdef CONFIG_PPC_ISERIES
 	if (firmware_has_feature(FW_FEATURE_ISERIES) &&
 			get_lppaca()->int_dword.fields.decr_int) {
 		get_lppaca()->int_dword.fields.decr_int = 0;
 		/* Signal a fake decrementer interrupt */
 		timer_interrupt(regs);
 	}
 #endif
 	trace_irq_exit(regs);
 }
 void __init init_IRQ(void)
 {
 	if (ppc_md.init_IRQ)
 		ppc_md.init_IRQ();
 	exc_lvl_ctx_init();
 	irq_ctx_init();
 }
 #if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
 struct thread_info   *critirq_ctx[NR_CPUS] __read_mostly;
 struct thread_info    *dbgirq_ctx[NR_CPUS] __read_mostly;
 struct thread_info *mcheckirq_ctx[NR_CPUS] __read_mostly;
 void exc_lvl_ctx_init(void)
 {
 	struct thread_info *tp;
 	int i;
 	for_each_possible_cpu(i) {
 		memset((void *)critirq_ctx[i], 0, THREAD_SIZE);
 		tp = critirq_ctx[i];
 		tp->cpu = i;
 		tp->preempt_count = 0;
 #ifdef CONFIG_BOOKE
 		memset((void *)dbgirq_ctx[i], 0, THREAD_SIZE);
 		tp = dbgirq_ctx[i];
 		tp->cpu = i;
 		tp->preempt_count = 0;
 		memset((void *)mcheckirq_ctx[i], 0, THREAD_SIZE);
 		tp = mcheckirq_ctx[i];
 		tp->cpu = i;
 		tp->preempt_count = HARDIRQ_OFFSET;
 #endif
 	}
 }
 #endif
 #ifdef CONFIG_IRQSTACKS
 struct thread_info *softirq_ctx[NR_CPUS] __read_mostly;
 struct thread_info *hardirq_ctx[NR_CPUS] __read_mostly;
 void irq_ctx_init(void)
 {
 	struct thread_info *tp;
 	int i;
 	for_each_possible_cpu(i) {
 		memset((void *)softirq_ctx[i], 0, THREAD_SIZE);
 		tp = softirq_ctx[i];
 		tp->cpu = i;
 		tp->preempt_count = 0;
 		memset((void *)hardirq_ctx[i], 0, THREAD_SIZE);
 		tp = hardirq_ctx[i];
 		tp->cpu = i;
 		tp->preempt_count = HARDIRQ_OFFSET;
 	}
 }
 static inline void do_softirq_onstack(void)
 {
 	struct thread_info *curtp, *irqtp;
 	unsigned long saved_sp_limit = current->thread.ksp_limit;
 	curtp = current_thread_info();
 	irqtp = softirq_ctx[smp_processor_id()];
 	irqtp->task = curtp->task;
 	current->thread.ksp_limit = (unsigned long)irqtp +
 				    _ALIGN_UP(sizeof(struct thread_info), 16);
 	call_do_softirq(irqtp);
 	current->thread.ksp_limit = saved_sp_limit;
 	irqtp->task = NULL;
 }
 #else
 #define do_softirq_onstack()	__do_softirq()
 #endif /* CONFIG_IRQSTACKS */
 void do_softirq(void)
 {
 	unsigned long flags;
 	if (in_interrupt())
 		return;
 	local_irq_save(flags);
 	if (local_softirq_pending())
 		do_softirq_onstack();
 	local_irq_restore(flags);
 }
 /*
  * IRQ controller and virtual interrupts
  */
 static LIST_HEAD(irq_hosts);
 static DEFINE_RAW_SPINLOCK(irq_big_lock);
 static unsigned int revmap_trees_allocated;
 static DEFINE_MUTEX(revmap_trees_mutex);
 struct irq_map_entry irq_map[NR_IRQS];
 static unsigned int irq_virq_count = NR_IRQS;
 static struct irq_host *irq_default_host;
 irq_hw_number_t virq_to_hw(unsigned int virq)
 {
 	return irq_map[virq].hwirq;
 }
 EXPORT_SYMBOL_GPL(virq_to_hw);
 static int default_irq_host_match(struct irq_host *h, struct device_node *np)
 {
 	return h->of_node != NULL && h->of_node == np;
 }
 struct irq_host *irq_alloc_host(struct device_node *of_node,
 				unsigned int revmap_type,
 				unsigned int revmap_arg,
 				struct irq_host_ops *ops,
 				irq_hw_number_t inval_irq)
 {
 	struct irq_host *host;
 	unsigned int size = sizeof(struct irq_host);
 	unsigned int i;
 	unsigned int *rmap;
 	unsigned long flags;
 	/* Allocate structure and revmap table if using linear mapping */
 	if (revmap_type == IRQ_HOST_MAP_LINEAR)
 		size += revmap_arg * sizeof(unsigned int);
 	host = zalloc_maybe_bootmem(size, GFP_KERNEL);
 	if (host == NULL)
 		return NULL;
 	/* Fill structure */
 	host->revmap_type = revmap_type;
 	host->inval_irq = inval_irq;
 	host->ops = ops;
 	host->of_node = of_node_get(of_node);
 	if (host->ops->match == NULL)
 		host->ops->match = default_irq_host_match;
 	raw_spin_lock_irqsave(&irq_big_lock, flags);
 	/* If it's a legacy controller, check for duplicates and
 	 * mark it as allocated (we use irq 0 host pointer for that
 	 */
 	if (revmap_type == IRQ_HOST_MAP_LEGACY) {
 		if (irq_map[0].host != NULL) {
 			raw_spin_unlock_irqrestore(&irq_big_lock, flags);
 			/* If we are early boot, we can't free the structure,
 			 * too bad...
 			 * this will be fixed once slab is made available early
 			 * instead of the current cruft
 			 */
 			if (mem_init_done)
 				kfree(host);
 			return NULL;
 		}
 		irq_map[0].host = host;
 	}
 	list_add(&host->link, &irq_hosts);
 	raw_spin_unlock_irqrestore(&irq_big_lock, flags);
 	/* Additional setups per revmap type */
 	switch(revmap_type) {
 	case IRQ_HOST_MAP_LEGACY:
 		/* 0 is always the invalid number for legacy */
 		host->inval_irq = 0;
 		/* setup us as the host for all legacy interrupts */
 		for (i = 1; i < NUM_ISA_INTERRUPTS; i++) {
 			irq_map[i].hwirq = i;
 			smp_wmb();
 			irq_map[i].host = host;
 			smp_wmb();
 			/* Clear norequest flags */
 			irq_to_desc(i)->status &= ~IRQ_NOREQUEST;
 			/* Legacy flags are left to default at this point,
 			 * one can then use irq_create_mapping() to
 			 * explicitly change them
 			 */
 			ops->map(host, i, i);
 		}
 		break;
 	case IRQ_HOST_MAP_LINEAR:
 		rmap = (unsigned int *)(host + 1);
 		for (i = 0; i < revmap_arg; i++)
 			rmap[i] = NO_IRQ;
 		host->revmap_data.linear.size = revmap_arg;
 		smp_wmb();
 		host->revmap_data.linear.revmap = rmap;
 		break;
 	default:
 		break;
 	}
 	pr_debug("irq: Allocated host of type %d @0x%p\n", revmap_type, host);
 	return host;
 }
 struct irq_host *irq_find_host(struct device_node *node)
 {
 	struct irq_host *h, *found = NULL;
 	unsigned long flags;
 	/* We might want to match the legacy controller last since
 	 * it might potentially be set to match all interrupts in
 	 * the absence of a device node. This isn't a problem so far
 	 * yet though...
 	 */
 	raw_spin_lock_irqsave(&irq_big_lock, flags);
 	list_for_each_entry(h, &irq_hosts, link)
 		if (h->ops->match(h, node)) {
 			found = h;
 			break;
 		}
 	raw_spin_unlock_irqrestore(&irq_big_lock, flags);
 	return found;
 }
 EXPORT_SYMBOL_GPL(irq_find_host);
 void irq_set_default_host(struct irq_host *host)
 {
 	pr_debug("irq: Default host set to @0x%p\n", host);
 	irq_default_host = host;
 }
 void irq_set_virq_count(unsigned int count)
 {
 	pr_debug("irq: Trying to set virq count to %d\n", count);
 	BUG_ON(count < NUM_ISA_INTERRUPTS);
 	if (count < NR_IRQS)
 		irq_virq_count = count;
 }
 static int irq_setup_virq(struct irq_host *host, unsigned int virq,
 			    irq_hw_number_t hwirq)
 {
 	struct irq_desc *desc;
 	desc = irq_to_desc_alloc_node(virq, 0);
 	if (!desc) {
 		pr_debug("irq: -> allocating desc failed\n");
 		goto error;
 	}
 	/* Clear IRQ_NOREQUEST flag */
 	desc->status &= ~IRQ_NOREQUEST;
 	/* map it */
 	smp_wmb();
 	irq_map[virq].hwirq = hwirq;
 	smp_mb();
 	if (host->ops->map(host, virq, hwirq)) {
 		pr_debug("irq: -> mapping failed, freeing\n");
 		goto error;
 	}
 	return 0;
 error:
 	irq_free_virt(virq, 1);
 	return -1;
 }
 unsigned int irq_create_direct_mapping(struct irq_host *host)
 {
 	unsigned int virq;
 	if (host == NULL)
 		host = irq_default_host;
 	BUG_ON(host == NULL);
 	WARN_ON(host->revmap_type != IRQ_HOST_MAP_NOMAP);
 	virq = irq_alloc_virt(host, 1, 0);
 	if (virq == NO_IRQ) {
 		pr_debug("irq: create_direct virq allocation failed\n");
 		return NO_IRQ;
 	}
 	pr_debug("irq: create_direct obtained virq %d\n", virq);
 	if (irq_setup_virq(host, virq, virq))
 		return NO_IRQ;
 	return virq;
 }
 unsigned int irq_create_mapping(struct irq_host *host,
 				irq_hw_number_t hwirq)
 {
 	unsigned int virq, hint;
 	pr_debug("irq: irq_create_mapping(0x%p, 0x%lx)\n", host, hwirq);
 	/* Look for default host if nececssary */
 	if (host == NULL)
 		host = irq_default_host;
 	if (host == NULL) {
 		printk(KERN_WARNING "irq_create_mapping called for"
 		       " NULL host, hwirq=%lx\n", hwirq);
 		WARN_ON(1);
 		return NO_IRQ;
 	}
 	pr_debug("irq: -> using host @%p\n", host);
 	/* Check if mapping already exist, if it does, call
 	 * host->ops->map() to update the flags
 	 */
 	virq = irq_find_mapping(host, hwirq);
 	if (virq != NO_IRQ) {
 		if (host->ops->remap)
 			host->ops->remap(host, virq, hwirq);
 		pr_debug("irq: -> existing mapping on virq %d\n", virq);
 		return virq;
 	}
 	/* Get a virtual interrupt number */
 	if (host->revmap_type == IRQ_HOST_MAP_LEGACY) {
 		/* Handle legacy */
 		virq = (unsigned int)hwirq;
 		if (virq == 0 || virq >= NUM_ISA_INTERRUPTS)
 			return NO_IRQ;
 		return virq;
 	} else {
 		/* Allocate a virtual interrupt number */
 		hint = hwirq % irq_virq_count;
 		virq = irq_alloc_virt(host, 1, hint);
 		if (virq == NO_IRQ) {
 			pr_debug("irq: -> virq allocation failed\n");
 			return NO_IRQ;
 		}
 	}
 	if (irq_setup_virq(host, virq, hwirq))
 		return NO_IRQ;
 	printk(KERN_DEBUG "irq: irq %lu on host %s mapped to virtual irq %u\n",
 		hwirq, host->of_node ? host->of_node->full_name : "null", virq);
 	return virq;
 }
 EXPORT_SYMBOL_GPL(irq_create_mapping);
 unsigned int irq_create_of_mapping(struct device_node *controller,
 				   const u32 *intspec, unsigned int intsize)
 {
 	struct irq_host *host;
 	irq_hw_number_t hwirq;
 	unsigned int type = IRQ_TYPE_NONE;
 	unsigned int virq;
 	if (controller == NULL)
 		host = irq_default_host;
 	else
 		host = irq_find_host(controller);
 	if (host == NULL) {
 		printk(KERN_WARNING "irq: no irq host found for %s !\n",
 		       controller->full_name);
 		return NO_IRQ;
 	}
 	/* If host has no translation, then we assume interrupt line */
 	if (host->ops->xlate == NULL)
 		hwirq = intspec[0];
 	else {
 		if (host->ops->xlate(host, controller, intspec, intsize,
 				     &hwirq, &type))
 			return NO_IRQ;
 	}
 	/* Create mapping */
 	virq = irq_create_mapping(host, hwirq);
 	if (virq == NO_IRQ)
 		return virq;
 	/* Set type if specified and different than the current one */
 	if (type != IRQ_TYPE_NONE &&
 	    type != (irq_to_desc(virq)->status & IRQF_TRIGGER_MASK))
 		set_irq_type(virq, type);
 	return virq;
 }
 EXPORT_SYMBOL_GPL(irq_create_of_mapping);
 unsigned int irq_of_parse_and_map(struct device_node *dev, int index)
 {
 	struct of_irq oirq;
 	if (of_irq_map_one(dev, index, &oirq))
 		return NO_IRQ;
 	return irq_create_of_mapping(oirq.controller, oirq.specifier,
 				     oirq.size);
 }
 EXPORT_SYMBOL_GPL(irq_of_parse_and_map);
 void irq_dispose_mapping(unsigned int virq)
 {
 	struct irq_host *host;
 	irq_hw_number_t hwirq;
 	if (virq == NO_IRQ)
 		return;
 	host = irq_map[virq].host;
 	WARN_ON (host == NULL);
 	if (host == NULL)
 		return;
 	/* Never unmap legacy interrupts */
 	if (host->revmap_type == IRQ_HOST_MAP_LEGACY)
 		return;
 	/* remove chip and handler */
 	set_irq_chip_and_handler(virq, NULL, NULL);
 	/* Make sure it's completed */
 	synchronize_irq(virq);
 	/* Tell the PIC about it */
 	if (host->ops->unmap)
 		host->ops->unmap(host, virq);
 	smp_mb();
 	/* Clear reverse map */
 	hwirq = irq_map[virq].hwirq;
 	switch(host->revmap_type) {
 	case IRQ_HOST_MAP_LINEAR:
 		if (hwirq < host->revmap_data.linear.size)
 			host->revmap_data.linear.revmap[hwirq] = NO_IRQ;
 		break;
 	case IRQ_HOST_MAP_TREE:
 		/*
 		 * Check if radix tree allocated yet, if not then nothing to
 		 * remove.
 		 */
 		smp_rmb();
 		if (revmap_trees_allocated < 1)
 			break;
 		mutex_lock(&revmap_trees_mutex);
 		radix_tree_delete(&host->revmap_data.tree, hwirq);
 		mutex_unlock(&revmap_trees_mutex);
 		break;
 	}
 	/* Destroy map */
 	smp_mb();
 	irq_map[virq].hwirq = host->inval_irq;
 	/* Set some flags */
 	irq_to_desc(virq)->status |= IRQ_NOREQUEST;
 	/* Free it */
 	irq_free_virt(virq, 1);
 }
 EXPORT_SYMBOL_GPL(irq_dispose_mapping);
 unsigned int irq_find_mapping(struct irq_host *host,
 			      irq_hw_number_t hwirq)
 {
 	unsigned int i;
 	unsigned int hint = hwirq % irq_virq_count;
 	/* Look for default host if nececssary */
 	if (host == NULL)
 		host = irq_default_host;
 	if (host == NULL)
 		return NO_IRQ;
 	/* legacy -> bail early */
 	if (host->revmap_type == IRQ_HOST_MAP_LEGACY)
 		return hwirq;
 	/* Slow path does a linear search of the map */
 	if (hint < NUM_ISA_INTERRUPTS)
 		hint = NUM_ISA_INTERRUPTS;
 	i = hint;
 	do  {
 		if (irq_map[i].host == host &&
 		    irq_map[i].hwirq == hwirq)
 			return i;
 		i++;
 		if (i >= irq_virq_count)
 			i = NUM_ISA_INTERRUPTS;
 	} while(i != hint);
 	return NO_IRQ;
 }
 EXPORT_SYMBOL_GPL(irq_find_mapping);
 unsigned int irq_radix_revmap_lookup(struct irq_host *host,
 				     irq_hw_number_t hwirq)
 {
 	struct irq_map_entry *ptr;
 	unsigned int virq;
 	WARN_ON(host->revmap_type != IRQ_HOST_MAP_TREE);
 	/*
 	 * Check if the radix tree exists and has bee initialized.
 	 * If not, we fallback to slow mode
 	 */
 	if (revmap_trees_allocated < 2)
 		return irq_find_mapping(host, hwirq);
 	/* Now try to resolve */
 	/*
 	 * No rcu_read_lock(ing) needed, the ptr returned can't go under us
 	 * as it's referencing an entry in the static irq_map table.
 	 */
 	ptr = radix_tree_lookup(&host->revmap_data.tree, hwirq);
 	/*
 	 * If found in radix tree, then fine.
 	 * Else fallback to linear lookup - this should not happen in practice
 	 * as it means that we failed to insert the node in the radix tree.
 	 */
 	if (ptr)
 		virq = ptr - irq_map;
 	else
 		virq = irq_find_mapping(host, hwirq);
 	return virq;
 }
 void irq_radix_revmap_insert(struct irq_host *host, unsigned int virq,
 			     irq_hw_number_t hwirq)
 {
 	WARN_ON(host->revmap_type != IRQ_HOST_MAP_TREE);
 	/*
 	 * Check if the radix tree exists yet.
 	 * If not, then the irq will be inserted into the tree when it gets
 	 * initialized.
 	 */
 	smp_rmb();
 	if (revmap_trees_allocated < 1)
 		return;
 	if (virq != NO_IRQ) {
 		mutex_lock(&revmap_trees_mutex);
 		radix_tree_insert(&host->revmap_data.tree, hwirq,
 				  &irq_map[virq]);
 		mutex_unlock(&revmap_trees_mutex);
 	}
 }
 unsigned int irq_linear_revmap(struct irq_host *host,
 			       irq_hw_number_t hwirq)
 {
 	unsigned int *revmap;
 	WARN_ON(host->revmap_type != IRQ_HOST_MAP_LINEAR);
 	/* Check revmap bounds */
 	if (unlikely(hwirq >= host->revmap_data.linear.size))
 		return irq_find_mapping(host, hwirq);
 	/* Check if revmap was allocated */
 	revmap = host->revmap_data.linear.revmap;
 	if (unlikely(revmap == NULL))
 		return irq_find_mapping(host, hwirq);
 	/* Fill up revmap with slow path if no mapping found */
 	if (unlikely(revmap[hwirq] == NO_IRQ))
 		revmap[hwirq] = irq_find_mapping(host, hwirq);
 	return revmap[hwirq];
 }
 unsigned int irq_alloc_virt(struct irq_host *host,
 			    unsigned int count,
 			    unsigned int hint)
 {
 	unsigned long flags;
 	unsigned int i, j, found = NO_IRQ;
 	if (count == 0 || count > (irq_virq_count - NUM_ISA_INTERRUPTS))
 		return NO_IRQ;
 	raw_spin_lock_irqsave(&irq_big_lock, flags);
 	/* Use hint for 1 interrupt if any */
 	if (count == 1 && hint >= NUM_ISA_INTERRUPTS &&
 	    hint < irq_virq_count && irq_map[hint].host == NULL) {
 		found = hint;
 		goto hint_found;
 	}
 	/* Look for count consecutive numbers in the allocatable
 	 * (non-legacy) space
 	 */
 	for (i = NUM_ISA_INTERRUPTS, j = 0; i < irq_virq_count; i++) {
 		if (irq_map[i].host != NULL)
 			j = 0;
 		else
 			j++;
 		if (j == count) {
 			found = i - count + 1;
 			break;
 		}
 	}
 	if (found == NO_IRQ) {
 		raw_spin_unlock_irqrestore(&irq_big_lock, flags);
 		return NO_IRQ;
 	}
  hint_found:
 	for (i = found; i < (found + count); i++) {
 		irq_map[i].hwirq = host->inval_irq;
 		smp_wmb();
 		irq_map[i].host = host;
 	}
 	raw_spin_unlock_irqrestore(&irq_big_lock, flags);
 	return found;
 }
 void irq_free_virt(unsigned int virq, unsigned int count)
 {
 	unsigned long flags;
 	unsigned int i;
 	WARN_ON (virq < NUM_ISA_INTERRUPTS);
 	WARN_ON (count == 0 || (virq + count) > irq_virq_count);
 	raw_spin_lock_irqsave(&irq_big_lock, flags);
 	for (i = virq; i < (virq + count); i++) {
 		struct irq_host *host;
 		if (i < NUM_ISA_INTERRUPTS ||
 		    (virq + count) > irq_virq_count)
 			continue;
 		host = irq_map[i].host;
 		irq_map[i].hwirq = host->inval_irq;
 		smp_wmb();
 		irq_map[i].host = NULL;
 	}
 	raw_spin_unlock_irqrestore(&irq_big_lock, flags);
 }
 int arch_early_irq_init(void)
 {
 	struct irq_desc *desc;
 	int i;
 	for (i = 0; i < NR_IRQS; i++) {
 		desc = irq_to_desc(i);
 		if (desc)
 			desc->status |= IRQ_NOREQUEST;
 	}
 	return 0;
 }
 int arch_init_chip_data(struct irq_desc *desc, int node)
 {
 	desc->status |= IRQ_NOREQUEST;
 	return 0;
 }
 /* We need to create the radix trees late */
 static int irq_late_init(void)
 {
 	struct irq_host *h;
 	unsigned int i;
 	/*
 	 * No mutual exclusion with respect to accessors of the tree is needed
 	 * here as the synchronization is done via the state variable
 	 * revmap_trees_allocated.
 	 */
 	list_for_each_entry(h, &irq_hosts, link) {
 		if (h->revmap_type == IRQ_HOST_MAP_TREE)
 			INIT_RADIX_TREE(&h->revmap_data.tree, GFP_KERNEL);
 	}
 	/*
 	 * Make sure the radix trees inits are visible before setting
 	 * the flag
 	 */
 	smp_wmb();
 	revmap_trees_allocated = 1;
 	/*
 	 * Insert the reverse mapping for those interrupts already present
 	 * in irq_map[].
 	 */
 	mutex_lock(&revmap_trees_mutex);
 	for (i = 0; i < irq_virq_count; i++) {
 		if (irq_map[i].host &&
 		    (irq_map[i].host->revmap_type == IRQ_HOST_MAP_TREE))
 			radix_tree_insert(&irq_map[i].host->revmap_data.tree,
 					  irq_map[i].hwirq, &irq_map[i]);
 	}
 	mutex_unlock(&revmap_trees_mutex);
 	/*
 	 * Make sure the radix trees insertions are visible before setting
 	 * the flag
 	 */
 	smp_wmb();
 	revmap_trees_allocated = 2;
 	return 0;
 }
 arch_initcall(irq_late_init);
 #ifdef CONFIG_VIRQ_DEBUG
 static int virq_debug_show(struct seq_file *m, void *private)
 {
 	unsigned long flags;
 	struct irq_desc *desc;
 	const char *p;
 	char none[] = "none";
 	int i;
 	seq_printf(m, "%-5s  %-7s  %-15s  %s\n", "virq", "hwirq",
 		      "chip name", "host name");
 	for (i = 1; i < nr_irqs; i++) {
 		desc = irq_to_desc(i);
 		if (!desc)
 			continue;
 		raw_spin_lock_irqsave(&desc->lock, flags);
 		if (desc->action && desc->action->handler) {
 			seq_printf(m, "%5d  ", i);
 			seq_printf(m, "0x%05lx  ", virq_to_hw(i));
 			if (desc->chip && desc->chip->name)
 				p = desc->chip->name;
 			else
 				p = none;
 			seq_printf(m, "%-15s  ", p);
 			if (irq_map[i].host && irq_map[i].host->of_node)
 				p = irq_map[i].host->of_node->full_name;
 			else
 				p = none;
 			seq_printf(m, "%s\n", p);
 		}
 		raw_spin_unlock_irqrestore(&desc->lock, flags);
 	}
 	return 0;
 }
 static int virq_debug_open(struct inode *inode, struct file *file)
 {
 	return single_open(file, virq_debug_show, inode->i_private);
 }
 static const struct file_operations virq_debug_fops = {
 	.open = virq_debug_open,
 	.read = seq_read,
 	.llseek = seq_lseek,
 	.release = single_release,
 };
 static int __init irq_debugfs_init(void)
 {
 	if (debugfs_create_file("virq_mapping", S_IRUGO, powerpc_debugfs_root,
 				 NULL, &virq_debug_fops) == NULL)
 		return -ENOMEM;
 	return 0;
 }
 __initcall(irq_debugfs_init);
 #endif /* CONFIG_VIRQ_DEBUG */
 #ifdef CONFIG_PPC64
 static int __init setup_noirqdistrib(char *str)
 {
 	distribute_irqs = 0;
 	return 1;
 }
 __setup("noirqdistrib", setup_noirqdistrib);
 #endif /* CONFIG_PPC64 */

arch/powerpc/kernel/time.c

Diff comments View file @ be83567

1	/*	1	/*
2	* Common time routines among all ppc machines.	2	* Common time routines among all ppc machines.
3	*	3	*
4	* Written by Cort Dougan (cort@cs.nmt.edu) to merge	4	* Written by Cort Dougan (cort@cs.nmt.edu) to merge
5	* Paul Mackerras' version and mine for PReP and Pmac.	5	* Paul Mackerras' version and mine for PReP and Pmac.
6	* MPC8xx/MBX changes by Dan Malek (dmalek@jlc.net).	6	* MPC8xx/MBX changes by Dan Malek (dmalek@jlc.net).
7	* Converted for 64-bit by Mike Corrigan (mikejc@us.ibm.com)	7	* Converted for 64-bit by Mike Corrigan (mikejc@us.ibm.com)
8	*	8	*
9	* First round of bugfixes by Gabriel Paubert (paubert@iram.es)	9	* First round of bugfixes by Gabriel Paubert (paubert@iram.es)
10	* to make clock more stable (2.4.0-test5). The only thing	10	* to make clock more stable (2.4.0-test5). The only thing
11	* that this code assumes is that the timebases have been synchronized	11	* that this code assumes is that the timebases have been synchronized
12	* by firmware on SMP and are never stopped (never do sleep	12	* by firmware on SMP and are never stopped (never do sleep
13	* on SMP then, nap and doze are OK).	13	* on SMP then, nap and doze are OK).
14	*	14	*
15	* Speeded up do_gettimeofday by getting rid of references to	15	* Speeded up do_gettimeofday by getting rid of references to
16	* xtime (which required locks for consistency). (mikejc@us.ibm.com)	16	* xtime (which required locks for consistency). (mikejc@us.ibm.com)
17	*	17	*
18	* TODO (not necessarily in this file):	18	* TODO (not necessarily in this file):
19	* - improve precision and reproducibility of timebase frequency	19	* - improve precision and reproducibility of timebase frequency
20	* measurement at boot time. (for iSeries, we calibrate the timebase	20	* measurement at boot time. (for iSeries, we calibrate the timebase
21	* against the Titan chip's clock.)	21	* against the Titan chip's clock.)
22	* - for astronomical applications: add a new function to get	22	* - for astronomical applications: add a new function to get
23	* non ambiguous timestamps even around leap seconds. This needs	23	* non ambiguous timestamps even around leap seconds. This needs
24	* a new timestamp format and a good name.	24	* a new timestamp format and a good name.
25	*	25	*
26	* 1997-09-10 Updated NTP code according to technical memorandum Jan '96	26	* 1997-09-10 Updated NTP code according to technical memorandum Jan '96
27	* "A Kernel Model for Precision Timekeeping" by Dave Mills	27	* "A Kernel Model for Precision Timekeeping" by Dave Mills
28	*	28	*
29	* This program is free software; you can redistribute it and/or	29	* This program is free software; you can redistribute it and/or
30	* modify it under the terms of the GNU General Public License	30	* modify it under the terms of the GNU General Public License
31	* as published by the Free Software Foundation; either version	31	* as published by the Free Software Foundation; either version
32	* 2 of the License, or (at your option) any later version.	32	* 2 of the License, or (at your option) any later version.
33	*/	33	*/
34		34
35	#include <linux/errno.h>	35	#include <linux/errno.h>
36	#include <linux/module.h>	36	#include <linux/module.h>
37	#include <linux/sched.h>	37	#include <linux/sched.h>
38	#include <linux/kernel.h>	38	#include <linux/kernel.h>
39	#include <linux/param.h>	39	#include <linux/param.h>
40	#include <linux/string.h>	40	#include <linux/string.h>
41	#include <linux/mm.h>	41	#include <linux/mm.h>
42	#include <linux/interrupt.h>	42	#include <linux/interrupt.h>
43	#include <linux/timex.h>	43	#include <linux/timex.h>
44	#include <linux/kernel_stat.h>	44	#include <linux/kernel_stat.h>
45	#include <linux/time.h>	45	#include <linux/time.h>
46	#include <linux/init.h>	46	#include <linux/init.h>
47	#include <linux/profile.h>	47	#include <linux/profile.h>
48	#include <linux/cpu.h>	48	#include <linux/cpu.h>
49	#include <linux/security.h>	49	#include <linux/security.h>
50	#include <linux/percpu.h>	50	#include <linux/percpu.h>
51	#include <linux/rtc.h>	51	#include <linux/rtc.h>
52	#include <linux/jiffies.h>	52	#include <linux/jiffies.h>
53	#include <linux/posix-timers.h>	53	#include <linux/posix-timers.h>
54	#include <linux/irq.h>	54	#include <linux/irq.h>
55	#include <linux/delay.h>	55	#include <linux/delay.h>
56	#include <linux/perf_event.h>	56	#include <linux/perf_event.h>
57	#include <asm/trace.h>	57	#include <asm/trace.h>
58		58
59	#include <asm/io.h>	59	#include <asm/io.h>
60	#include <asm/processor.h>	60	#include <asm/processor.h>
61	#include <asm/nvram.h>	61	#include <asm/nvram.h>
62	#include <asm/cache.h>	62	#include <asm/cache.h>
63	#include <asm/machdep.h>	63	#include <asm/machdep.h>
64	#include <asm/uaccess.h>	64	#include <asm/uaccess.h>
65	#include <asm/time.h>	65	#include <asm/time.h>
66	#include <asm/prom.h>	66	#include <asm/prom.h>
67	#include <asm/irq.h>	67	#include <asm/irq.h>
68	#include <asm/div64.h>	68	#include <asm/div64.h>
69	#include <asm/smp.h>	69	#include <asm/smp.h>
70	#include <asm/vdso_datapage.h>	70	#include <asm/vdso_datapage.h>
71	#include <asm/firmware.h>	71	#include <asm/firmware.h>
72	#include <asm/cputime.h>	72	#include <asm/cputime.h>
73	#ifdef CONFIG_PPC_ISERIES	73	#ifdef CONFIG_PPC_ISERIES
74	#include <asm/iseries/it_lp_queue.h>	74	#include <asm/iseries/it_lp_queue.h>
75	#include <asm/iseries/hv_call_xm.h>	75	#include <asm/iseries/hv_call_xm.h>
76	#endif	76	#endif
77		77
78	/* powerpc clocksource/clockevent code */	78	/* powerpc clocksource/clockevent code */
79		79
80	#include <linux/clockchips.h>	80	#include <linux/clockchips.h>
81	#include <linux/clocksource.h>	81	#include <linux/clocksource.h>
82		82
83	static cycle_t rtc_read(struct clocksource *);	83	static cycle_t rtc_read(struct clocksource *);
84	static struct clocksource clocksource_rtc = {	84	static struct clocksource clocksource_rtc = {
85	.name = "rtc",	85	.name = "rtc",
86	.rating = 400,	86	.rating = 400,
87	.flags = CLOCK_SOURCE_IS_CONTINUOUS,	87	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
88	.mask = CLOCKSOURCE_MASK(64),	88	.mask = CLOCKSOURCE_MASK(64),
89	.shift = 22,	89	.shift = 22,
90	.mult = 0, /* To be filled in */	90	.mult = 0, /* To be filled in */
91	.read = rtc_read,	91	.read = rtc_read,
92	};	92	};
93		93
94	static cycle_t timebase_read(struct clocksource *);	94	static cycle_t timebase_read(struct clocksource *);
95	static struct clocksource clocksource_timebase = {	95	static struct clocksource clocksource_timebase = {
96	.name = "timebase",	96	.name = "timebase",
97	.rating = 400,	97	.rating = 400,
98	.flags = CLOCK_SOURCE_IS_CONTINUOUS,	98	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
99	.mask = CLOCKSOURCE_MASK(64),	99	.mask = CLOCKSOURCE_MASK(64),
100	.shift = 22,	100	.shift = 22,
101	.mult = 0, /* To be filled in */	101	.mult = 0, /* To be filled in */
102	.read = timebase_read,	102	.read = timebase_read,
103	};	103	};
104		104
105	#define DECREMENTER_MAX 0x7fffffff	105	#define DECREMENTER_MAX 0x7fffffff
106		106
107	static int decrementer_set_next_event(unsigned long evt,	107	static int decrementer_set_next_event(unsigned long evt,
108	struct clock_event_device *dev);	108	struct clock_event_device *dev);
109	static void decrementer_set_mode(enum clock_event_mode mode,	109	static void decrementer_set_mode(enum clock_event_mode mode,
110	struct clock_event_device *dev);	110	struct clock_event_device *dev);
111		111
112	static struct clock_event_device decrementer_clockevent = {	112	static struct clock_event_device decrementer_clockevent = {
113	.name = "decrementer",	113	.name = "decrementer",
114	.rating = 200,	114	.rating = 200,
115	.shift = 0, /* To be filled in */	115	.shift = 0, /* To be filled in */
116	.mult = 0, /* To be filled in */	116	.mult = 0, /* To be filled in */
117	.irq = 0,	117	.irq = 0,
118	.set_next_event = decrementer_set_next_event,	118	.set_next_event = decrementer_set_next_event,
119	.set_mode = decrementer_set_mode,	119	.set_mode = decrementer_set_mode,
120	.features = CLOCK_EVT_FEAT_ONESHOT,	120	.features = CLOCK_EVT_FEAT_ONESHOT,
121	};	121	};
122		122
123	struct decrementer_clock {	123	struct decrementer_clock {
124	struct clock_event_device event;	124	struct clock_event_device event;
125	u64 next_tb;	125	u64 next_tb;
126	};	126	};
127		127
128	static DEFINE_PER_CPU(struct decrementer_clock, decrementers);	128	static DEFINE_PER_CPU(struct decrementer_clock, decrementers);
129		129
130	#ifdef CONFIG_PPC_ISERIES	130	#ifdef CONFIG_PPC_ISERIES
131	static unsigned long __initdata iSeries_recal_titan;	131	static unsigned long __initdata iSeries_recal_titan;
132	static signed long __initdata iSeries_recal_tb;	132	static signed long __initdata iSeries_recal_tb;
133		133
134	/* Forward declaration is only needed for iSereis compiles */	134	/* Forward declaration is only needed for iSereis compiles */
135	static void __init clocksource_init(void);	135	static void __init clocksource_init(void);
136	#endif	136	#endif
137		137
138	#define XSEC_PER_SEC (1024*1024)	138	#define XSEC_PER_SEC (1024*1024)
139		139
140	#ifdef CONFIG_PPC64	140	#ifdef CONFIG_PPC64
141	#define SCALE_XSEC(xsec, max) (((xsec) * max) / XSEC_PER_SEC)	141	#define SCALE_XSEC(xsec, max) (((xsec) * max) / XSEC_PER_SEC)
142	#else	142	#else
143	/* compute ((xsec << 12) * max) >> 32 */	143	/* compute ((xsec << 12) * max) >> 32 */
144	#define SCALE_XSEC(xsec, max) mulhwu((xsec) << 12, max)	144	#define SCALE_XSEC(xsec, max) mulhwu((xsec) << 12, max)
145	#endif	145	#endif
146		146
147	unsigned long tb_ticks_per_jiffy;	147	unsigned long tb_ticks_per_jiffy;
148	unsigned long tb_ticks_per_usec = 100; /* sane default */	148	unsigned long tb_ticks_per_usec = 100; /* sane default */
149	EXPORT_SYMBOL(tb_ticks_per_usec);	149	EXPORT_SYMBOL(tb_ticks_per_usec);
150	unsigned long tb_ticks_per_sec;	150	unsigned long tb_ticks_per_sec;
151	EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime_t conversions */	151	EXPORT_SYMBOL(tb_ticks_per_sec); /* for cputime_t conversions */
152	u64 tb_to_xs;	152	u64 tb_to_xs;
153	unsigned tb_to_us;	153	unsigned tb_to_us;
154		154
155	#define TICKLEN_SCALE NTP_SCALE_SHIFT	155	#define TICKLEN_SCALE NTP_SCALE_SHIFT
156	static u64 last_tick_len; /* units are ns / 2^TICKLEN_SCALE */	156	static u64 last_tick_len; /* units are ns / 2^TICKLEN_SCALE */
157	static u64 ticklen_to_xs; /* 0.64 fraction */	157	static u64 ticklen_to_xs; /* 0.64 fraction */
158		158
159	/* If last_tick_len corresponds to about 1/HZ seconds, then	159	/* If last_tick_len corresponds to about 1/HZ seconds, then
160	last_tick_len << TICKLEN_SHIFT will be about 2^63. */	160	last_tick_len << TICKLEN_SHIFT will be about 2^63. */
161	#define TICKLEN_SHIFT (63 - 30 - TICKLEN_SCALE + SHIFT_HZ)	161	#define TICKLEN_SHIFT (63 - 30 - TICKLEN_SCALE + SHIFT_HZ)
162		162
163	DEFINE_SPINLOCK(rtc_lock);	163	DEFINE_SPINLOCK(rtc_lock);
164	EXPORT_SYMBOL_GPL(rtc_lock);	164	EXPORT_SYMBOL_GPL(rtc_lock);
165		165
166	static u64 tb_to_ns_scale __read_mostly;	166	static u64 tb_to_ns_scale __read_mostly;
167	static unsigned tb_to_ns_shift __read_mostly;	167	static unsigned tb_to_ns_shift __read_mostly;
168	static unsigned long boot_tb __read_mostly;	168	static unsigned long boot_tb __read_mostly;
169		169
170	extern struct timezone sys_tz;	170	extern struct timezone sys_tz;
171	static long timezone_offset;	171	static long timezone_offset;
172		172
173	unsigned long ppc_proc_freq;	173	unsigned long ppc_proc_freq;
174	EXPORT_SYMBOL(ppc_proc_freq);	174	EXPORT_SYMBOL(ppc_proc_freq);
175	unsigned long ppc_tb_freq;	175	unsigned long ppc_tb_freq;
176		176
177	static u64 tb_last_jiffy __cacheline_aligned_in_smp;	177	static u64 tb_last_jiffy __cacheline_aligned_in_smp;
178	static DEFINE_PER_CPU(u64, last_jiffy);	178	static DEFINE_PER_CPU(u64, last_jiffy);
179		179
180	#ifdef CONFIG_VIRT_CPU_ACCOUNTING	180	#ifdef CONFIG_VIRT_CPU_ACCOUNTING
181	/*	181	/*
182	* Factors for converting from cputime_t (timebase ticks) to	182	* Factors for converting from cputime_t (timebase ticks) to
183	* jiffies, milliseconds, seconds, and clock_t (1/USER_HZ seconds).	183	* jiffies, milliseconds, seconds, and clock_t (1/USER_HZ seconds).
184	* These are all stored as 0.64 fixed-point binary fractions.	184	* These are all stored as 0.64 fixed-point binary fractions.
185	*/	185	*/
186	u64 __cputime_jiffies_factor;	186	u64 __cputime_jiffies_factor;
187	EXPORT_SYMBOL(__cputime_jiffies_factor);	187	EXPORT_SYMBOL(__cputime_jiffies_factor);
188	u64 __cputime_msec_factor;	188	u64 __cputime_msec_factor;
189	EXPORT_SYMBOL(__cputime_msec_factor);	189	EXPORT_SYMBOL(__cputime_msec_factor);
190	u64 __cputime_sec_factor;	190	u64 __cputime_sec_factor;
191	EXPORT_SYMBOL(__cputime_sec_factor);	191	EXPORT_SYMBOL(__cputime_sec_factor);
192	u64 __cputime_clockt_factor;	192	u64 __cputime_clockt_factor;
193	EXPORT_SYMBOL(__cputime_clockt_factor);	193	EXPORT_SYMBOL(__cputime_clockt_factor);
194	DEFINE_PER_CPU(unsigned long, cputime_last_delta);	194	DEFINE_PER_CPU(unsigned long, cputime_last_delta);
195	DEFINE_PER_CPU(unsigned long, cputime_scaled_last_delta);	195	DEFINE_PER_CPU(unsigned long, cputime_scaled_last_delta);
196		196
197	cputime_t cputime_one_jiffy;	197	cputime_t cputime_one_jiffy;
198		198
199	static void calc_cputime_factors(void)	199	static void calc_cputime_factors(void)
200	{	200	{
201	struct div_result res;	201	struct div_result res;
202		202
203	div128_by_32(HZ, 0, tb_ticks_per_sec, &res);	203	div128_by_32(HZ, 0, tb_ticks_per_sec, &res);
204	__cputime_jiffies_factor = res.result_low;	204	__cputime_jiffies_factor = res.result_low;
205	div128_by_32(1000, 0, tb_ticks_per_sec, &res);	205	div128_by_32(1000, 0, tb_ticks_per_sec, &res);
206	__cputime_msec_factor = res.result_low;	206	__cputime_msec_factor = res.result_low;
207	div128_by_32(1, 0, tb_ticks_per_sec, &res);	207	div128_by_32(1, 0, tb_ticks_per_sec, &res);
208	__cputime_sec_factor = res.result_low;	208	__cputime_sec_factor = res.result_low;
209	div128_by_32(USER_HZ, 0, tb_ticks_per_sec, &res);	209	div128_by_32(USER_HZ, 0, tb_ticks_per_sec, &res);
210	__cputime_clockt_factor = res.result_low;	210	__cputime_clockt_factor = res.result_low;
211	}	211	}
212		212
213	/*	213	/*
214	* Read the PURR on systems that have it, otherwise the timebase.	214	* Read the PURR on systems that have it, otherwise the timebase.
215	*/	215	*/
216	static u64 read_purr(void)	216	static u64 read_purr(void)
217	{	217	{
218	if (cpu_has_feature(CPU_FTR_PURR))	218	if (cpu_has_feature(CPU_FTR_PURR))
219	return mfspr(SPRN_PURR);	219	return mfspr(SPRN_PURR);
220	return mftb();	220	return mftb();
221	}	221	}
222		222
223	/*	223	/*
224	* Read the SPURR on systems that have it, otherwise the purr	224	* Read the SPURR on systems that have it, otherwise the purr
225	*/	225	*/
226	static u64 read_spurr(u64 purr)	226	static u64 read_spurr(u64 purr)
227	{	227	{
228	/*	228	/*
229	* cpus without PURR won't have a SPURR	229	* cpus without PURR won't have a SPURR
230	* We already know the former when we use this, so tell gcc	230	* We already know the former when we use this, so tell gcc
231	*/	231	*/
232	if (cpu_has_feature(CPU_FTR_PURR) && cpu_has_feature(CPU_FTR_SPURR))	232	if (cpu_has_feature(CPU_FTR_PURR) && cpu_has_feature(CPU_FTR_SPURR))
233	return mfspr(SPRN_SPURR);	233	return mfspr(SPRN_SPURR);
234	return purr;	234	return purr;
235	}	235	}
236		236
237	/*	237	/*
238	* Account time for a transition between system, hard irq	238	* Account time for a transition between system, hard irq
239	* or soft irq state.	239	* or soft irq state.
240	*/	240	*/
241	void account_system_vtime(struct task_struct *tsk)	241	void account_system_vtime(struct task_struct *tsk)
242	{	242	{
243	u64 now, nowscaled, delta, deltascaled, sys_time;	243	u64 now, nowscaled, delta, deltascaled, sys_time;
244	unsigned long flags;	244	unsigned long flags;
245		245
246	local_irq_save(flags);	246	local_irq_save(flags);
247	now = read_purr();	247	now = read_purr();
248	nowscaled = read_spurr(now);	248	nowscaled = read_spurr(now);
249	delta = now - get_paca()->startpurr;	249	delta = now - get_paca()->startpurr;
250	deltascaled = nowscaled - get_paca()->startspurr;	250	deltascaled = nowscaled - get_paca()->startspurr;
251	get_paca()->startpurr = now;	251	get_paca()->startpurr = now;
252	get_paca()->startspurr = nowscaled;	252	get_paca()->startspurr = nowscaled;
253	if (!in_interrupt()) {	253	if (!in_interrupt()) {
254	/* deltascaled includes both user and system time.	254	/* deltascaled includes both user and system time.
255	* Hence scale it based on the purr ratio to estimate	255	* Hence scale it based on the purr ratio to estimate
256	* the system time */	256	* the system time */
257	sys_time = get_paca()->system_time;	257	sys_time = get_paca()->system_time;
258	if (get_paca()->user_time)	258	if (get_paca()->user_time)
259	deltascaled = deltascaled * sys_time /	259	deltascaled = deltascaled * sys_time /
260	(sys_time + get_paca()->user_time);	260	(sys_time + get_paca()->user_time);
261	delta += sys_time;	261	delta += sys_time;
262	get_paca()->system_time = 0;	262	get_paca()->system_time = 0;
263	}	263	}
264	if (in_irq() \|\| idle_task(smp_processor_id()) != tsk)	264	if (in_irq() \|\| idle_task(smp_processor_id()) != tsk)
265	account_system_time(tsk, 0, delta, deltascaled);	265	account_system_time(tsk, 0, delta, deltascaled);
266	else	266	else
267	account_idle_time(delta);	267	account_idle_time(delta);
268	__get_cpu_var(cputime_last_delta) = delta;	268	__get_cpu_var(cputime_last_delta) = delta;
269	__get_cpu_var(cputime_scaled_last_delta) = deltascaled;	269	__get_cpu_var(cputime_scaled_last_delta) = deltascaled;
270	local_irq_restore(flags);	270	local_irq_restore(flags);
271	}	271	}
272	EXPORT_SYMBOL_GPL(account_system_vtime);	272	EXPORT_SYMBOL_GPL(account_system_vtime);
273		273
274	/*	274	/*
275	* Transfer the user and system times accumulated in the paca	275	* Transfer the user and system times accumulated in the paca
276	* by the exception entry and exit code to the generic process	276	* by the exception entry and exit code to the generic process
277	* user and system time records.	277	* user and system time records.
278	* Must be called with interrupts disabled.	278	* Must be called with interrupts disabled.
279	*/	279	*/
280	void account_process_tick(struct task_struct *tsk, int user_tick)	280	void account_process_tick(struct task_struct *tsk, int user_tick)
281	{	281	{
282	cputime_t utime, utimescaled;	282	cputime_t utime, utimescaled;
283		283
284	utime = get_paca()->user_time;	284	utime = get_paca()->user_time;
285	get_paca()->user_time = 0;	285	get_paca()->user_time = 0;
286	utimescaled = cputime_to_scaled(utime);	286	utimescaled = cputime_to_scaled(utime);
287	account_user_time(tsk, utime, utimescaled);	287	account_user_time(tsk, utime, utimescaled);
288	}	288	}
289		289
290	/*	290	/*
291	* Stuff for accounting stolen time.	291	* Stuff for accounting stolen time.
292	*/	292	*/
293	struct cpu_purr_data {	293	struct cpu_purr_data {
294	int initialized; /* thread is running */	294	int initialized; /* thread is running */
295	u64 tb; /* last TB value read */	295	u64 tb; /* last TB value read */
296	u64 purr; /* last PURR value read */	296	u64 purr; /* last PURR value read */
297	u64 spurr; /* last SPURR value read */	297	u64 spurr; /* last SPURR value read */
298	};	298	};
299		299
300	/*	300	/*
301	* Each entry in the cpu_purr_data array is manipulated only by its	301	* Each entry in the cpu_purr_data array is manipulated only by its
302	* "owner" cpu -- usually in the timer interrupt but also occasionally	302	* "owner" cpu -- usually in the timer interrupt but also occasionally
303	* in process context for cpu online. As long as cpus do not touch	303	* in process context for cpu online. As long as cpus do not touch
304	* each others' cpu_purr_data, disabling local interrupts is	304	* each others' cpu_purr_data, disabling local interrupts is
305	* sufficient to serialize accesses.	305	* sufficient to serialize accesses.
306	*/	306	*/
307	static DEFINE_PER_CPU(struct cpu_purr_data, cpu_purr_data);	307	static DEFINE_PER_CPU(struct cpu_purr_data, cpu_purr_data);
308		308
309	static void snapshot_tb_and_purr(void *data)	309	static void snapshot_tb_and_purr(void *data)
310	{	310	{
311	unsigned long flags;	311	unsigned long flags;
312	struct cpu_purr_data *p = &__get_cpu_var(cpu_purr_data);	312	struct cpu_purr_data *p = &__get_cpu_var(cpu_purr_data);
313		313
314	local_irq_save(flags);	314	local_irq_save(flags);
315	p->tb = get_tb_or_rtc();	315	p->tb = get_tb_or_rtc();
316	p->purr = mfspr(SPRN_PURR);	316	p->purr = mfspr(SPRN_PURR);
317	wmb();	317	wmb();
318	p->initialized = 1;	318	p->initialized = 1;
319	local_irq_restore(flags);	319	local_irq_restore(flags);
320	}	320	}
321		321
322	/*	322	/*
323	* Called during boot when all cpus have come up.	323	* Called during boot when all cpus have come up.
324	*/	324	*/
325	void snapshot_timebases(void)	325	void snapshot_timebases(void)
326	{	326	{
327	if (!cpu_has_feature(CPU_FTR_PURR))	327	if (!cpu_has_feature(CPU_FTR_PURR))
328	return;	328	return;
329	on_each_cpu(snapshot_tb_and_purr, NULL, 1);	329	on_each_cpu(snapshot_tb_and_purr, NULL, 1);
330	}	330	}
331		331
332	/*	332	/*
333	* Must be called with interrupts disabled.	333	* Must be called with interrupts disabled.
334	*/	334	*/
335	void calculate_steal_time(void)	335	void calculate_steal_time(void)
336	{	336	{
337	u64 tb, purr;	337	u64 tb, purr;
338	s64 stolen;	338	s64 stolen;
339	struct cpu_purr_data *pme;	339	struct cpu_purr_data *pme;
340		340
341	pme = &__get_cpu_var(cpu_purr_data);	341	pme = &__get_cpu_var(cpu_purr_data);
342	if (!pme->initialized)	342	if (!pme->initialized)
343	return; /* !CPU_FTR_PURR or early in early boot */	343	return; /* !CPU_FTR_PURR or early in early boot */
344	tb = mftb();	344	tb = mftb();
345	purr = mfspr(SPRN_PURR);	345	purr = mfspr(SPRN_PURR);
346	stolen = (tb - pme->tb) - (purr - pme->purr);	346	stolen = (tb - pme->tb) - (purr - pme->purr);
347	if (stolen > 0) {	347	if (stolen > 0) {
348	if (idle_task(smp_processor_id()) != current)	348	if (idle_task(smp_processor_id()) != current)
349	account_steal_time(stolen);	349	account_steal_time(stolen);
350	else	350	else
351	account_idle_time(stolen);	351	account_idle_time(stolen);
352	}	352	}
353	pme->tb = tb;	353	pme->tb = tb;
354	pme->purr = purr;	354	pme->purr = purr;
355	}	355	}
356		356
357	#ifdef CONFIG_PPC_SPLPAR	357	#ifdef CONFIG_PPC_SPLPAR
358	/*	358	/*
359	* Must be called before the cpu is added to the online map when	359	* Must be called before the cpu is added to the online map when
360	* a cpu is being brought up at runtime.	360	* a cpu is being brought up at runtime.
361	*/	361	*/
362	static void snapshot_purr(void)	362	static void snapshot_purr(void)
363	{	363	{
364	struct cpu_purr_data *pme;	364	struct cpu_purr_data *pme;
365	unsigned long flags;	365	unsigned long flags;
366		366
367	if (!cpu_has_feature(CPU_FTR_PURR))	367	if (!cpu_has_feature(CPU_FTR_PURR))
368	return;	368	return;
369	local_irq_save(flags);	369	local_irq_save(flags);
370	pme = &__get_cpu_var(cpu_purr_data);	370	pme = &__get_cpu_var(cpu_purr_data);
371	pme->tb = mftb();	371	pme->tb = mftb();
372	pme->purr = mfspr(SPRN_PURR);	372	pme->purr = mfspr(SPRN_PURR);
373	pme->initialized = 1;	373	pme->initialized = 1;
374	local_irq_restore(flags);	374	local_irq_restore(flags);
375	}	375	}
376		376
377	#endif /* CONFIG_PPC_SPLPAR */	377	#endif /* CONFIG_PPC_SPLPAR */
378		378
379	#else /* ! CONFIG_VIRT_CPU_ACCOUNTING */	379	#else /* ! CONFIG_VIRT_CPU_ACCOUNTING */
380	#define calc_cputime_factors()	380	#define calc_cputime_factors()
381	#define calculate_steal_time() do { } while (0)	381	#define calculate_steal_time() do { } while (0)
382	#endif	382	#endif
383		383
384	#if !(defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR))	384	#if !(defined(CONFIG_VIRT_CPU_ACCOUNTING) && defined(CONFIG_PPC_SPLPAR))
385	#define snapshot_purr() do { } while (0)	385	#define snapshot_purr() do { } while (0)
386	#endif	386	#endif
387		387
388	/*	388	/*
389	* Called when a cpu comes up after the system has finished booting,	389	* Called when a cpu comes up after the system has finished booting,
390	* i.e. as a result of a hotplug cpu action.	390	* i.e. as a result of a hotplug cpu action.
391	*/	391	*/
392	void snapshot_timebase(void)	392	void snapshot_timebase(void)
393	{	393	{
394	__get_cpu_var(last_jiffy) = get_tb_or_rtc();	394	__get_cpu_var(last_jiffy) = get_tb_or_rtc();
395	snapshot_purr();	395	snapshot_purr();
396	}	396	}
397		397
398	void __delay(unsigned long loops)	398	void __delay(unsigned long loops)
399	{	399	{
400	unsigned long start;	400	unsigned long start;
401	int diff;	401	int diff;
402		402
403	if (__USE_RTC()) {	403	if (__USE_RTC()) {
404	start = get_rtcl();	404	start = get_rtcl();
405	do {	405	do {
406	/* the RTCL register wraps at 1000000000 */	406	/* the RTCL register wraps at 1000000000 */
407	diff = get_rtcl() - start;	407	diff = get_rtcl() - start;
408	if (diff < 0)	408	if (diff < 0)
409	diff += 1000000000;	409	diff += 1000000000;
410	} while (diff < loops);	410	} while (diff < loops);
411	} else {	411	} else {
412	start = get_tbl();	412	start = get_tbl();
413	while (get_tbl() - start < loops)	413	while (get_tbl() - start < loops)
414	HMT_low();	414	HMT_low();
415	HMT_medium();	415	HMT_medium();
416	}	416	}
417	}	417	}
418	EXPORT_SYMBOL(__delay);	418	EXPORT_SYMBOL(__delay);
419		419
420	void udelay(unsigned long usecs)	420	void udelay(unsigned long usecs)
421	{	421	{
422	__delay(tb_ticks_per_usec * usecs);	422	__delay(tb_ticks_per_usec * usecs);
423	}	423	}
424	EXPORT_SYMBOL(udelay);	424	EXPORT_SYMBOL(udelay);
425		425
426	static inline void update_gtod(u64 new_tb_stamp, u64 new_stamp_xsec,	426	static inline void update_gtod(u64 new_tb_stamp, u64 new_stamp_xsec,
427	u64 new_tb_to_xs)	427	u64 new_tb_to_xs)
428	{	428	{
429	/*	429	/*
430	* tb_update_count is used to allow the userspace gettimeofday code	430	* tb_update_count is used to allow the userspace gettimeofday code
431	* to assure itself that it sees a consistent view of the tb_to_xs and	431	* to assure itself that it sees a consistent view of the tb_to_xs and
432	* stamp_xsec variables. It reads the tb_update_count, then reads	432	* stamp_xsec variables. It reads the tb_update_count, then reads
433	* tb_to_xs and stamp_xsec and then reads tb_update_count again. If	433	* tb_to_xs and stamp_xsec and then reads tb_update_count again. If
434	* the two values of tb_update_count match and are even then the	434	* the two values of tb_update_count match and are even then the
435	* tb_to_xs and stamp_xsec values are consistent. If not, then it	435	* tb_to_xs and stamp_xsec values are consistent. If not, then it
436	* loops back and reads them again until this criteria is met.	436	* loops back and reads them again until this criteria is met.
437	* We expect the caller to have done the first increment of	437	* We expect the caller to have done the first increment of
438	* vdso_data->tb_update_count already.	438	* vdso_data->tb_update_count already.
439	*/	439	*/
440	vdso_data->tb_orig_stamp = new_tb_stamp;	440	vdso_data->tb_orig_stamp = new_tb_stamp;
441	vdso_data->stamp_xsec = new_stamp_xsec;	441	vdso_data->stamp_xsec = new_stamp_xsec;
442	vdso_data->tb_to_xs = new_tb_to_xs;	442	vdso_data->tb_to_xs = new_tb_to_xs;
443	vdso_data->wtom_clock_sec = wall_to_monotonic.tv_sec;	443	vdso_data->wtom_clock_sec = wall_to_monotonic.tv_sec;
444	vdso_data->wtom_clock_nsec = wall_to_monotonic.tv_nsec;	444	vdso_data->wtom_clock_nsec = wall_to_monotonic.tv_nsec;
445	vdso_data->stamp_xtime = xtime;	445	vdso_data->stamp_xtime = xtime;
446	smp_wmb();	446	smp_wmb();
447	++(vdso_data->tb_update_count);	447	++(vdso_data->tb_update_count);
448	}	448	}
449		449
450	#ifdef CONFIG_SMP	450	#ifdef CONFIG_SMP
451	unsigned long profile_pc(struct pt_regs *regs)	451	unsigned long profile_pc(struct pt_regs *regs)
452	{	452	{
453	unsigned long pc = instruction_pointer(regs);	453	unsigned long pc = instruction_pointer(regs);
454		454
455	if (in_lock_functions(pc))	455	if (in_lock_functions(pc))
456	return regs->link;	456	return regs->link;
457		457
458	return pc;	458	return pc;
459	}	459	}
460	EXPORT_SYMBOL(profile_pc);	460	EXPORT_SYMBOL(profile_pc);
461	#endif	461	#endif
462		462
463	#ifdef CONFIG_PPC_ISERIES	463	#ifdef CONFIG_PPC_ISERIES
464		464
465	/*	465	/*
466	* This function recalibrates the timebase based on the 49-bit time-of-day	466	* This function recalibrates the timebase based on the 49-bit time-of-day
467	* value in the Titan chip. The Titan is much more accurate than the value	467	* value in the Titan chip. The Titan is much more accurate than the value
468	* returned by the service processor for the timebase frequency.	468	* returned by the service processor for the timebase frequency.
469	*/	469	*/
470		470
471	static int __init iSeries_tb_recal(void)	471	static int __init iSeries_tb_recal(void)
472	{	472	{
473	struct div_result divres;	473	struct div_result divres;
474	unsigned long titan, tb;	474	unsigned long titan, tb;
475		475
476	/* Make sure we only run on iSeries */	476	/* Make sure we only run on iSeries */
477	if (!firmware_has_feature(FW_FEATURE_ISERIES))	477	if (!firmware_has_feature(FW_FEATURE_ISERIES))
478	return -ENODEV;	478	return -ENODEV;
479		479
480	tb = get_tb();	480	tb = get_tb();
481	titan = HvCallXm_loadTod();	481	titan = HvCallXm_loadTod();
482	if ( iSeries_recal_titan ) {	482	if ( iSeries_recal_titan ) {
483	unsigned long tb_ticks = tb - iSeries_recal_tb;	483	unsigned long tb_ticks = tb - iSeries_recal_tb;
484	unsigned long titan_usec = (titan - iSeries_recal_titan) >> 12;	484	unsigned long titan_usec = (titan - iSeries_recal_titan) >> 12;
485	unsigned long new_tb_ticks_per_sec = (tb_ticks * USEC_PER_SEC)/titan_usec;	485	unsigned long new_tb_ticks_per_sec = (tb_ticks * USEC_PER_SEC)/titan_usec;
486	unsigned long new_tb_ticks_per_jiffy =	486	unsigned long new_tb_ticks_per_jiffy =
487	DIV_ROUND_CLOSEST(new_tb_ticks_per_sec, HZ);	487	DIV_ROUND_CLOSEST(new_tb_ticks_per_sec, HZ);
488	long tick_diff = new_tb_ticks_per_jiffy - tb_ticks_per_jiffy;	488	long tick_diff = new_tb_ticks_per_jiffy - tb_ticks_per_jiffy;
489	char sign = '+';	489	char sign = '+';
490	/* make sure tb_ticks_per_sec and tb_ticks_per_jiffy are consistent */	490	/* make sure tb_ticks_per_sec and tb_ticks_per_jiffy are consistent */
491	new_tb_ticks_per_sec = new_tb_ticks_per_jiffy * HZ;	491	new_tb_ticks_per_sec = new_tb_ticks_per_jiffy * HZ;
492		492
493	if ( tick_diff < 0 ) {	493	if ( tick_diff < 0 ) {
494	tick_diff = -tick_diff;	494	tick_diff = -tick_diff;
495	sign = '-';	495	sign = '-';
496	}	496	}
497	if ( tick_diff ) {	497	if ( tick_diff ) {
498	if ( tick_diff < tb_ticks_per_jiffy/25 ) {	498	if ( tick_diff < tb_ticks_per_jiffy/25 ) {
499	printk( "Titan recalibrate: new tb_ticks_per_jiffy = %lu (%c%ld)\n",	499	printk( "Titan recalibrate: new tb_ticks_per_jiffy = %lu (%c%ld)\n",
500	new_tb_ticks_per_jiffy, sign, tick_diff );	500	new_tb_ticks_per_jiffy, sign, tick_diff );
501	tb_ticks_per_jiffy = new_tb_ticks_per_jiffy;	501	tb_ticks_per_jiffy = new_tb_ticks_per_jiffy;
502	tb_ticks_per_sec = new_tb_ticks_per_sec;	502	tb_ticks_per_sec = new_tb_ticks_per_sec;
503	calc_cputime_factors();	503	calc_cputime_factors();
504	div128_by_32( XSEC_PER_SEC, 0, tb_ticks_per_sec, &divres );	504	div128_by_32( XSEC_PER_SEC, 0, tb_ticks_per_sec, &divres );
505	tb_to_xs = divres.result_low;	505	tb_to_xs = divres.result_low;
506	vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;	506	vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
507	vdso_data->tb_to_xs = tb_to_xs;	507	vdso_data->tb_to_xs = tb_to_xs;
508	setup_cputime_one_jiffy();	508	setup_cputime_one_jiffy();
509	}	509	}
510	else {	510	else {
511	printk( "Titan recalibrate: FAILED (difference > 4 percent)\n"	511	printk( "Titan recalibrate: FAILED (difference > 4 percent)\n"
512	" new tb_ticks_per_jiffy = %lu\n"	512	" new tb_ticks_per_jiffy = %lu\n"
513	" old tb_ticks_per_jiffy = %lu\n",	513	" old tb_ticks_per_jiffy = %lu\n",
514	new_tb_ticks_per_jiffy, tb_ticks_per_jiffy );	514	new_tb_ticks_per_jiffy, tb_ticks_per_jiffy );
515	}	515	}
516	}	516	}
517	}	517	}
518	iSeries_recal_titan = titan;	518	iSeries_recal_titan = titan;
519	iSeries_recal_tb = tb;	519	iSeries_recal_tb = tb;
520		520
521	/* Called here as now we know accurate values for the timebase */	521	/* Called here as now we know accurate values for the timebase */
522	clocksource_init();	522	clocksource_init();
523	return 0;	523	return 0;
524	}	524	}
525	late_initcall(iSeries_tb_recal);	525	late_initcall(iSeries_tb_recal);
526		526
527	/* Called from platform early init */	527	/* Called from platform early init */
528	void __init iSeries_time_init_early(void)	528	void __init iSeries_time_init_early(void)
529	{	529	{
530	iSeries_recal_tb = get_tb();	530	iSeries_recal_tb = get_tb();
531	iSeries_recal_titan = HvCallXm_loadTod();	531	iSeries_recal_titan = HvCallXm_loadTod();
532	}	532	}
533	#endif /* CONFIG_PPC_ISERIES */	533	#endif /* CONFIG_PPC_ISERIES */
534		534
535	#if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_PPC32)	535	#ifdef CONFIG_PERF_EVENTS
536	DEFINE_PER_CPU(u8, perf_event_pending);
537		536
538	void set_perf_event_pending(void)	537	/*
		538	* 64-bit uses a byte in the PACA, 32-bit uses a per-cpu variable...
		539	*/
		540	#ifdef CONFIG_PPC64
		541	static inline unsigned long test_perf_event_pending(void)
539	{	542	{
540	get_cpu_var(perf_event_pending) = 1;	543	unsigned long x;
541	set_dec(1);	544
542	put_cpu_var(perf_event_pending);	545	asm volatile("lbz %0,%1(13)"
		546	: "=r" (x)
		547	: "i" (offsetof(struct paca_struct, perf_event_pending)));
		548	return x;
543	}	549	}
544		550
		551	static inline void set_perf_event_pending_flag(void)
		552	{
		553	asm volatile("stb %0,%1(13)" : :
		554	"r" (1),
		555	"i" (offsetof(struct paca_struct, perf_event_pending)));
		556	}
		557
		558	static inline void clear_perf_event_pending(void)
		559	{
		560	asm volatile("stb %0,%1(13)" : :
		561	"r" (0),
		562	"i" (offsetof(struct paca_struct, perf_event_pending)));
		563	}
		564
		565	#else /* 32-bit */
		566
		567	DEFINE_PER_CPU(u8, perf_event_pending);
		568
		569	#define set_perf_event_pending_flag() __get_cpu_var(perf_event_pending) = 1
545	#define test_perf_event_pending() __get_cpu_var(perf_event_pending)	570	#define test_perf_event_pending() __get_cpu_var(perf_event_pending)
546	#define clear_perf_event_pending() __get_cpu_var(perf_event_pending) = 0	571	#define clear_perf_event_pending() __get_cpu_var(perf_event_pending) = 0
547		572
548	#else /* CONFIG_PERF_EVENTS && CONFIG_PPC32 */	573	#endif /* 32 vs 64 bit */
549		574
		575	void set_perf_event_pending(void)
		576	{
		577	preempt_disable();
		578	set_perf_event_pending_flag();
		579	set_dec(1);
		580	preempt_enable();
		581	}
		582
		583	#else /* CONFIG_PERF_EVENTS */
		584
550	#define test_perf_event_pending() 0	585	#define test_perf_event_pending() 0
551	#define clear_perf_event_pending()	586	#define clear_perf_event_pending()
552		587
553	#endif /* CONFIG_PERF_EVENTS && CONFIG_PPC32 */	588	#endif /* CONFIG_PERF_EVENTS */
554		589
555	/*	590	/*
556	* For iSeries shared processors, we have to let the hypervisor	591	* For iSeries shared processors, we have to let the hypervisor
557	* set the hardware decrementer. We set a virtual decrementer	592	* set the hardware decrementer. We set a virtual decrementer
558	* in the lppaca and call the hypervisor if the virtual	593	* in the lppaca and call the hypervisor if the virtual
559	* decrementer is less than the current value in the hardware	594	* decrementer is less than the current value in the hardware
560	* decrementer. (almost always the new decrementer value will	595	* decrementer. (almost always the new decrementer value will
561	* be greater than the current hardware decementer so the hypervisor	596	* be greater than the current hardware decementer so the hypervisor
562	* call will not be needed)	597	* call will not be needed)
563	*/	598	*/
564		599
565	/*	600	/*
566	* timer_interrupt - gets called when the decrementer overflows,	601	* timer_interrupt - gets called when the decrementer overflows,
567	* with interrupts disabled.	602	* with interrupts disabled.
568	*/	603	*/
569	void timer_interrupt(struct pt_regs * regs)	604	void timer_interrupt(struct pt_regs * regs)
570	{	605	{
571	struct pt_regs *old_regs;	606	struct pt_regs *old_regs;
572	struct decrementer_clock *decrementer = &__get_cpu_var(decrementers);	607	struct decrementer_clock *decrementer = &__get_cpu_var(decrementers);
573	struct clock_event_device *evt = &decrementer->event;	608	struct clock_event_device *evt = &decrementer->event;
574	u64 now;	609	u64 now;
575		610
576	trace_timer_interrupt_entry(regs);	611	trace_timer_interrupt_entry(regs);
577		612
578	__get_cpu_var(irq_stat).timer_irqs++;	613	__get_cpu_var(irq_stat).timer_irqs++;
579		614
580	/* Ensure a positive value is written to the decrementer, or else	615	/* Ensure a positive value is written to the decrementer, or else
581	* some CPUs will continuue to take decrementer exceptions */	616	* some CPUs will continuue to take decrementer exceptions */
582	set_dec(DECREMENTER_MAX);	617	set_dec(DECREMENTER_MAX);
583		618
584	#ifdef CONFIG_PPC32	619	#ifdef CONFIG_PPC32
585	if (test_perf_event_pending()) {
586	clear_perf_event_pending();
587	perf_event_do_pending();
588	}
589	if (atomic_read(&ppc_n_lost_interrupts) != 0)	620	if (atomic_read(&ppc_n_lost_interrupts) != 0)
590	do_IRQ(regs);	621	do_IRQ(regs);
591	#endif	622	#endif
592		623
593	now = get_tb_or_rtc();	624	now = get_tb_or_rtc();
594	if (now < decrementer->next_tb) {	625	if (now < decrementer->next_tb) {
595	/* not time for this event yet */	626	/* not time for this event yet */
596	now = decrementer->next_tb - now;	627	now = decrementer->next_tb - now;
597	if (now <= DECREMENTER_MAX)	628	if (now <= DECREMENTER_MAX)
598	set_dec((int)now);	629	set_dec((int)now);
599	trace_timer_interrupt_exit(regs);	630	trace_timer_interrupt_exit(regs);
600	return;	631	return;
601	}	632	}
602	old_regs = set_irq_regs(regs);	633	old_regs = set_irq_regs(regs);
603	irq_enter();	634	irq_enter();
604		635
605	calculate_steal_time();	636	calculate_steal_time();
		637
		638	if (test_perf_event_pending()) {
		639	clear_perf_event_pending();
		640	perf_event_do_pending();
		641	}
606		642
607	#ifdef CONFIG_PPC_ISERIES	643	#ifdef CONFIG_PPC_ISERIES
608	if (firmware_has_feature(FW_FEATURE_ISERIES))	644	if (firmware_has_feature(FW_FEATURE_ISERIES))
609	get_lppaca()->int_dword.fields.decr_int = 0;	645	get_lppaca()->int_dword.fields.decr_int = 0;
610	#endif	646	#endif
611		647
612	if (evt->event_handler)	648	if (evt->event_handler)
613	evt->event_handler(evt);	649	evt->event_handler(evt);
614		650
615	#ifdef CONFIG_PPC_ISERIES	651	#ifdef CONFIG_PPC_ISERIES
616	if (firmware_has_feature(FW_FEATURE_ISERIES) && hvlpevent_is_pending())	652	if (firmware_has_feature(FW_FEATURE_ISERIES) && hvlpevent_is_pending())
617	process_hvlpevents();	653	process_hvlpevents();
618	#endif	654	#endif
619		655
620	#ifdef CONFIG_PPC64	656	#ifdef CONFIG_PPC64
621	/* collect purr register values often, for accurate calculations */	657	/* collect purr register values often, for accurate calculations */
622	if (firmware_has_feature(FW_FEATURE_SPLPAR)) {	658	if (firmware_has_feature(FW_FEATURE_SPLPAR)) {
623	struct cpu_usage *cu = &__get_cpu_var(cpu_usage_array);	659	struct cpu_usage *cu = &__get_cpu_var(cpu_usage_array);
624	cu->current_tb = mfspr(SPRN_PURR);	660	cu->current_tb = mfspr(SPRN_PURR);
625	}	661	}
626	#endif	662	#endif
627		663
628	irq_exit();	664	irq_exit();
629	set_irq_regs(old_regs);	665	set_irq_regs(old_regs);
630		666
631	trace_timer_interrupt_exit(regs);	667	trace_timer_interrupt_exit(regs);
632	}	668	}
633		669
634	void wakeup_decrementer(void)	670	void wakeup_decrementer(void)
635	{	671	{
636	unsigned long ticks;	672	unsigned long ticks;
637		673
638	/*	674	/*
639	* The timebase gets saved on sleep and restored on wakeup,	675	* The timebase gets saved on sleep and restored on wakeup,
640	* so all we need to do is to reset the decrementer.	676	* so all we need to do is to reset the decrementer.
641	*/	677	*/
642	ticks = tb_ticks_since(__get_cpu_var(last_jiffy));	678	ticks = tb_ticks_since(__get_cpu_var(last_jiffy));
643	if (ticks < tb_ticks_per_jiffy)	679	if (ticks < tb_ticks_per_jiffy)
644	ticks = tb_ticks_per_jiffy - ticks;	680	ticks = tb_ticks_per_jiffy - ticks;
645	else	681	else
646	ticks = 1;	682	ticks = 1;
647	set_dec(ticks);	683	set_dec(ticks);
648	}	684	}
649		685
650	#ifdef CONFIG_SUSPEND	686	#ifdef CONFIG_SUSPEND
651	void generic_suspend_disable_irqs(void)	687	void generic_suspend_disable_irqs(void)
652	{	688	{
653	preempt_disable();	689	preempt_disable();
654		690
655	/* Disable the decrementer, so that it doesn't interfere	691	/* Disable the decrementer, so that it doesn't interfere
656	* with suspending.	692	* with suspending.
657	*/	693	*/
658		694
659	set_dec(0x7fffffff);	695	set_dec(0x7fffffff);
660	local_irq_disable();	696	local_irq_disable();
661	set_dec(0x7fffffff);	697	set_dec(0x7fffffff);
662	}	698	}
663		699
664	void generic_suspend_enable_irqs(void)	700	void generic_suspend_enable_irqs(void)
665	{	701	{
666	wakeup_decrementer();	702	wakeup_decrementer();
667		703
668	local_irq_enable();	704	local_irq_enable();
669	preempt_enable();	705	preempt_enable();
670	}	706	}
671		707
672	/* Overrides the weak version in kernel/power/main.c */	708	/* Overrides the weak version in kernel/power/main.c */
673	void arch_suspend_disable_irqs(void)	709	void arch_suspend_disable_irqs(void)
674	{	710	{
675	if (ppc_md.suspend_disable_irqs)	711	if (ppc_md.suspend_disable_irqs)
676	ppc_md.suspend_disable_irqs();	712	ppc_md.suspend_disable_irqs();
677	generic_suspend_disable_irqs();	713	generic_suspend_disable_irqs();
678	}	714	}
679		715
680	/* Overrides the weak version in kernel/power/main.c */	716	/* Overrides the weak version in kernel/power/main.c */
681	void arch_suspend_enable_irqs(void)	717	void arch_suspend_enable_irqs(void)
682	{	718	{
683	generic_suspend_enable_irqs();	719	generic_suspend_enable_irqs();
684	if (ppc_md.suspend_enable_irqs)	720	if (ppc_md.suspend_enable_irqs)
685	ppc_md.suspend_enable_irqs();	721	ppc_md.suspend_enable_irqs();
686	}	722	}
687	#endif	723	#endif
688		724
689	#ifdef CONFIG_SMP	725	#ifdef CONFIG_SMP
690	void __init smp_space_timers(unsigned int max_cpus)	726	void __init smp_space_timers(unsigned int max_cpus)
691	{	727	{
692	int i;	728	int i;
693	u64 previous_tb = per_cpu(last_jiffy, boot_cpuid);	729	u64 previous_tb = per_cpu(last_jiffy, boot_cpuid);
694		730
695	/* make sure tb > per_cpu(last_jiffy, cpu) for all cpus always */	731	/* make sure tb > per_cpu(last_jiffy, cpu) for all cpus always */
696	previous_tb -= tb_ticks_per_jiffy;	732	previous_tb -= tb_ticks_per_jiffy;
697		733
698	for_each_possible_cpu(i) {	734	for_each_possible_cpu(i) {
699	if (i == boot_cpuid)	735	if (i == boot_cpuid)
700	continue;	736	continue;
701	per_cpu(last_jiffy, i) = previous_tb;	737	per_cpu(last_jiffy, i) = previous_tb;
702	}	738	}
703	}	739	}
704	#endif	740	#endif
705		741
706	/*	742	/*
707	* Scheduler clock - returns current time in nanosec units.	743	* Scheduler clock - returns current time in nanosec units.
708	*	744	*
709	* Note: mulhdu(a, b) (multiply high double unsigned) returns	745	* Note: mulhdu(a, b) (multiply high double unsigned) returns
710	* the high 64 bits of a * b, i.e. (a * b) >> 64, where a and b	746	* the high 64 bits of a * b, i.e. (a * b) >> 64, where a and b
711	* are 64-bit unsigned numbers.	747	* are 64-bit unsigned numbers.
712	*/	748	*/
713	unsigned long long sched_clock(void)	749	unsigned long long sched_clock(void)
714	{	750	{
715	if (__USE_RTC())	751	if (__USE_RTC())
716	return get_rtc();	752	return get_rtc();
717	return mulhdu(get_tb() - boot_tb, tb_to_ns_scale) << tb_to_ns_shift;	753	return mulhdu(get_tb() - boot_tb, tb_to_ns_scale) << tb_to_ns_shift;
718	}	754	}
719		755
720	static int __init get_freq(char name, int cells, unsigned long val)	756	static int __init get_freq(char name, int cells, unsigned long val)
721	{	757	{
722	struct device_node *cpu;	758	struct device_node *cpu;
723	const unsigned int *fp;	759	const unsigned int *fp;
724	int found = 0;	760	int found = 0;
725		761
726	/* The cpu node should have timebase and clock frequency properties */	762	/* The cpu node should have timebase and clock frequency properties */
727	cpu = of_find_node_by_type(NULL, "cpu");	763	cpu = of_find_node_by_type(NULL, "cpu");
728		764
729	if (cpu) {	765	if (cpu) {
730	fp = of_get_property(cpu, name, NULL);	766	fp = of_get_property(cpu, name, NULL);
731	if (fp) {	767	if (fp) {
732	found = 1;	768	found = 1;
733	*val = of_read_ulong(fp, cells);	769	*val = of_read_ulong(fp, cells);
734	}	770	}
735		771
736	of_node_put(cpu);	772	of_node_put(cpu);
737	}	773	}
738		774
739	return found;	775	return found;
740	}	776	}
741		777
742	/* should become __cpuinit when secondary_cpu_time_init also is */	778	/* should become __cpuinit when secondary_cpu_time_init also is */
743	void start_cpu_decrementer(void)	779	void start_cpu_decrementer(void)
744	{	780	{
745	#if defined(CONFIG_BOOKE) \|\| defined(CONFIG_40x)	781	#if defined(CONFIG_BOOKE) \|\| defined(CONFIG_40x)
746	/* Clear any pending timer interrupts */	782	/* Clear any pending timer interrupts */
747	mtspr(SPRN_TSR, TSR_ENW \| TSR_WIS \| TSR_DIS \| TSR_FIS);	783	mtspr(SPRN_TSR, TSR_ENW \| TSR_WIS \| TSR_DIS \| TSR_FIS);
748		784
749	/* Enable decrementer interrupt */	785	/* Enable decrementer interrupt */
750	mtspr(SPRN_TCR, TCR_DIE);	786	mtspr(SPRN_TCR, TCR_DIE);
751	#endif /* defined(CONFIG_BOOKE) \|\| defined(CONFIG_40x) */	787	#endif /* defined(CONFIG_BOOKE) \|\| defined(CONFIG_40x) */
752	}	788	}
753		789
754	void __init generic_calibrate_decr(void)	790	void __init generic_calibrate_decr(void)
755	{	791	{
756	ppc_tb_freq = DEFAULT_TB_FREQ; /* hardcoded default */	792	ppc_tb_freq = DEFAULT_TB_FREQ; /* hardcoded default */
757		793
758	if (!get_freq("ibm,extended-timebase-frequency", 2, &ppc_tb_freq) &&	794	if (!get_freq("ibm,extended-timebase-frequency", 2, &ppc_tb_freq) &&
759	!get_freq("timebase-frequency", 1, &ppc_tb_freq)) {	795	!get_freq("timebase-frequency", 1, &ppc_tb_freq)) {
760		796
761	printk(KERN_ERR "WARNING: Estimating decrementer frequency "	797	printk(KERN_ERR "WARNING: Estimating decrementer frequency "
762	"(not found)\n");	798	"(not found)\n");
763	}	799	}
764		800
765	ppc_proc_freq = DEFAULT_PROC_FREQ; /* hardcoded default */	801	ppc_proc_freq = DEFAULT_PROC_FREQ; /* hardcoded default */
766		802
767	if (!get_freq("ibm,extended-clock-frequency", 2, &ppc_proc_freq) &&	803	if (!get_freq("ibm,extended-clock-frequency", 2, &ppc_proc_freq) &&
768	!get_freq("clock-frequency", 1, &ppc_proc_freq)) {	804	!get_freq("clock-frequency", 1, &ppc_proc_freq)) {
769		805
770	printk(KERN_ERR "WARNING: Estimating processor frequency "	806	printk(KERN_ERR "WARNING: Estimating processor frequency "
771	"(not found)\n");	807	"(not found)\n");
772	}	808	}
773	}	809	}
774		810
775	int update_persistent_clock(struct timespec now)	811	int update_persistent_clock(struct timespec now)
776	{	812	{
777	struct rtc_time tm;	813	struct rtc_time tm;
778		814
779	if (!ppc_md.set_rtc_time)	815	if (!ppc_md.set_rtc_time)
780	return 0;	816	return 0;
781		817
782	to_tm(now.tv_sec + 1 + timezone_offset, &tm);	818	to_tm(now.tv_sec + 1 + timezone_offset, &tm);
783	tm.tm_year -= 1900;	819	tm.tm_year -= 1900;
784	tm.tm_mon -= 1;	820	tm.tm_mon -= 1;
785		821
786	return ppc_md.set_rtc_time(&tm);	822	return ppc_md.set_rtc_time(&tm);
787	}	823	}
788		824
789	static void __read_persistent_clock(struct timespec *ts)	825	static void __read_persistent_clock(struct timespec *ts)
790	{	826	{
791	struct rtc_time tm;	827	struct rtc_time tm;
792	static int first = 1;	828	static int first = 1;
793		829
794	ts->tv_nsec = 0;	830	ts->tv_nsec = 0;
795	/* XXX this is a litle fragile but will work okay in the short term */	831	/* XXX this is a litle fragile but will work okay in the short term */
796	if (first) {	832	if (first) {
797	first = 0;	833	first = 0;
798	if (ppc_md.time_init)	834	if (ppc_md.time_init)
799	timezone_offset = ppc_md.time_init();	835	timezone_offset = ppc_md.time_init();
800		836
801	/* get_boot_time() isn't guaranteed to be safe to call late */	837	/* get_boot_time() isn't guaranteed to be safe to call late */
802	if (ppc_md.get_boot_time) {	838	if (ppc_md.get_boot_time) {
803	ts->tv_sec = ppc_md.get_boot_time() - timezone_offset;	839	ts->tv_sec = ppc_md.get_boot_time() - timezone_offset;
804	return;	840	return;
805	}	841	}
806	}	842	}
807	if (!ppc_md.get_rtc_time) {	843	if (!ppc_md.get_rtc_time) {
808	ts->tv_sec = 0;	844	ts->tv_sec = 0;
809	return;	845	return;
810	}	846	}
811	ppc_md.get_rtc_time(&tm);	847	ppc_md.get_rtc_time(&tm);
812		848
813	ts->tv_sec = mktime(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday,	849	ts->tv_sec = mktime(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday,
814	tm.tm_hour, tm.tm_min, tm.tm_sec);	850	tm.tm_hour, tm.tm_min, tm.tm_sec);
815	}	851	}
816		852
817	void read_persistent_clock(struct timespec *ts)	853	void read_persistent_clock(struct timespec *ts)
818	{	854	{
819	__read_persistent_clock(ts);	855	__read_persistent_clock(ts);
820		856
821	/* Sanitize it in case real time clock is set below EPOCH */	857	/* Sanitize it in case real time clock is set below EPOCH */
822	if (ts->tv_sec < 0) {	858	if (ts->tv_sec < 0) {
823	ts->tv_sec = 0;	859	ts->tv_sec = 0;
824	ts->tv_nsec = 0;	860	ts->tv_nsec = 0;
825	}	861	}
826		862
827	}	863	}
828		864
829	/* clocksource code */	865	/* clocksource code */
830	static cycle_t rtc_read(struct clocksource *cs)	866	static cycle_t rtc_read(struct clocksource *cs)
831	{	867	{
832	return (cycle_t)get_rtc();	868	return (cycle_t)get_rtc();
833	}	869	}
834		870
835	static cycle_t timebase_read(struct clocksource *cs)	871	static cycle_t timebase_read(struct clocksource *cs)
836	{	872	{
837	return (cycle_t)get_tb();	873	return (cycle_t)get_tb();
838	}	874	}
839		875
840	void update_vsyscall(struct timespec wall_time, struct clocksource clock,	876	void update_vsyscall(struct timespec wall_time, struct clocksource clock,
841	u32 mult)	877	u32 mult)
842	{	878	{
843	u64 t2x, stamp_xsec;	879	u64 t2x, stamp_xsec;
844		880
845	if (clock != &clocksource_timebase)	881	if (clock != &clocksource_timebase)
846	return;	882	return;
847		883
848	/* Make userspace gettimeofday spin until we're done. */	884	/* Make userspace gettimeofday spin until we're done. */
849	++vdso_data->tb_update_count;	885	++vdso_data->tb_update_count;
850	smp_mb();	886	smp_mb();
851		887
852	/* XXX this assumes clock->shift == 22 */	888	/* XXX this assumes clock->shift == 22 */
853	/* 4611686018 ~= 2^(20+64-22) / 1e9 */	889	/* 4611686018 ~= 2^(20+64-22) / 1e9 */
854	t2x = (u64) mult * 4611686018ULL;	890	t2x = (u64) mult * 4611686018ULL;
855	stamp_xsec = (u64) xtime.tv_nsec * XSEC_PER_SEC;	891	stamp_xsec = (u64) xtime.tv_nsec * XSEC_PER_SEC;
856	do_div(stamp_xsec, 1000000000);	892	do_div(stamp_xsec, 1000000000);
857	stamp_xsec += (u64) xtime.tv_sec * XSEC_PER_SEC;	893	stamp_xsec += (u64) xtime.tv_sec * XSEC_PER_SEC;
858	update_gtod(clock->cycle_last, stamp_xsec, t2x);	894	update_gtod(clock->cycle_last, stamp_xsec, t2x);
859	}	895	}
860		896
861	void update_vsyscall_tz(void)	897	void update_vsyscall_tz(void)
862	{	898	{
863	/* Make userspace gettimeofday spin until we're done. */	899	/* Make userspace gettimeofday spin until we're done. */
864	++vdso_data->tb_update_count;	900	++vdso_data->tb_update_count;
865	smp_mb();	901	smp_mb();
866	vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;	902	vdso_data->tz_minuteswest = sys_tz.tz_minuteswest;
867	vdso_data->tz_dsttime = sys_tz.tz_dsttime;	903	vdso_data->tz_dsttime = sys_tz.tz_dsttime;
868	smp_mb();	904	smp_mb();
869	++vdso_data->tb_update_count;	905	++vdso_data->tb_update_count;
870	}	906	}
871		907
872	static void __init clocksource_init(void)	908	static void __init clocksource_init(void)
873	{	909	{
874	struct clocksource *clock;	910	struct clocksource *clock;
875		911
876	if (__USE_RTC())	912	if (__USE_RTC())
877	clock = &clocksource_rtc;	913	clock = &clocksource_rtc;
878	else	914	else
879	clock = &clocksource_timebase;	915	clock = &clocksource_timebase;
880		916
881	clock->mult = clocksource_hz2mult(tb_ticks_per_sec, clock->shift);	917	clock->mult = clocksource_hz2mult(tb_ticks_per_sec, clock->shift);
882		918
883	if (clocksource_register(clock)) {	919	if (clocksource_register(clock)) {
884	printk(KERN_ERR "clocksource: %s is already registered\n",	920	printk(KERN_ERR "clocksource: %s is already registered\n",
885	clock->name);	921	clock->name);
886	return;	922	return;
887	}	923	}
888		924
889	printk(KERN_INFO "clocksource: %s mult[%x] shift[%d] registered\n",	925	printk(KERN_INFO "clocksource: %s mult[%x] shift[%d] registered\n",
890	clock->name, clock->mult, clock->shift);	926	clock->name, clock->mult, clock->shift);
891	}	927	}
892		928
893	static int decrementer_set_next_event(unsigned long evt,	929	static int decrementer_set_next_event(unsigned long evt,
894	struct clock_event_device *dev)	930	struct clock_event_device *dev)
895	{	931	{
896	__get_cpu_var(decrementers).next_tb = get_tb_or_rtc() + evt;	932	__get_cpu_var(decrementers).next_tb = get_tb_or_rtc() + evt;
897	set_dec(evt);	933	set_dec(evt);
898	return 0;	934	return 0;
899	}	935	}
900		936
901	static void decrementer_set_mode(enum clock_event_mode mode,	937	static void decrementer_set_mode(enum clock_event_mode mode,
902	struct clock_event_device *dev)	938	struct clock_event_device *dev)
903	{	939	{
904	if (mode != CLOCK_EVT_MODE_ONESHOT)	940	if (mode != CLOCK_EVT_MODE_ONESHOT)
905	decrementer_set_next_event(DECREMENTER_MAX, dev);	941	decrementer_set_next_event(DECREMENTER_MAX, dev);
906	}	942	}
907		943
908	static inline uint64_t div_sc64(unsigned long ticks, unsigned long nsec,	944	static inline uint64_t div_sc64(unsigned long ticks, unsigned long nsec,
909	int shift)	945	int shift)
910	{	946	{
911	uint64_t tmp = ((uint64_t)ticks) << shift;	947	uint64_t tmp = ((uint64_t)ticks) << shift;
912		948
913	do_div(tmp, nsec);	949	do_div(tmp, nsec);
914	return tmp;	950	return tmp;
915	}	951	}
916		952
917	static void __init setup_clockevent_multiplier(unsigned long hz)	953	static void __init setup_clockevent_multiplier(unsigned long hz)
918	{	954	{
919	u64 mult, shift = 32;	955	u64 mult, shift = 32;
920		956
921	while (1) {	957	while (1) {
922	mult = div_sc64(hz, NSEC_PER_SEC, shift);	958	mult = div_sc64(hz, NSEC_PER_SEC, shift);
923	if (mult && (mult >> 32UL) == 0UL)	959	if (mult && (mult >> 32UL) == 0UL)
924	break;	960	break;
925		961
926	shift--;	962	shift--;
927	}	963	}
928		964
929	decrementer_clockevent.shift = shift;	965	decrementer_clockevent.shift = shift;
930	decrementer_clockevent.mult = mult;	966	decrementer_clockevent.mult = mult;
931	}	967	}
932		968
933	static void register_decrementer_clockevent(int cpu)	969	static void register_decrementer_clockevent(int cpu)
934	{	970	{
935	struct clock_event_device *dec = &per_cpu(decrementers, cpu).event;	971	struct clock_event_device *dec = &per_cpu(decrementers, cpu).event;
936		972
937	*dec = decrementer_clockevent;	973	*dec = decrementer_clockevent;
938	dec->cpumask = cpumask_of(cpu);	974	dec->cpumask = cpumask_of(cpu);
939		975
940	printk_once(KERN_DEBUG "clockevent: %s mult[%x] shift[%d] cpu[%d]\n",	976	printk_once(KERN_DEBUG "clockevent: %s mult[%x] shift[%d] cpu[%d]\n",
941	dec->name, dec->mult, dec->shift, cpu);	977	dec->name, dec->mult, dec->shift, cpu);
942		978
943	clockevents_register_device(dec);	979	clockevents_register_device(dec);
944	}	980	}
945		981
946	static void __init init_decrementer_clockevent(void)	982	static void __init init_decrementer_clockevent(void)
947	{	983	{
948	int cpu = smp_processor_id();	984	int cpu = smp_processor_id();
949		985
950	setup_clockevent_multiplier(ppc_tb_freq);	986	setup_clockevent_multiplier(ppc_tb_freq);
951	decrementer_clockevent.max_delta_ns =	987	decrementer_clockevent.max_delta_ns =
952	clockevent_delta2ns(DECREMENTER_MAX, &decrementer_clockevent);	988	clockevent_delta2ns(DECREMENTER_MAX, &decrementer_clockevent);
953	decrementer_clockevent.min_delta_ns =	989	decrementer_clockevent.min_delta_ns =
954	clockevent_delta2ns(2, &decrementer_clockevent);	990	clockevent_delta2ns(2, &decrementer_clockevent);
955		991
956	register_decrementer_clockevent(cpu);	992	register_decrementer_clockevent(cpu);
957	}	993	}
958		994
959	void secondary_cpu_time_init(void)	995	void secondary_cpu_time_init(void)
960	{	996	{
961	/* Start the decrementer on CPUs that have manual control	997	/* Start the decrementer on CPUs that have manual control
962	* such as BookE	998	* such as BookE
963	*/	999	*/
964	start_cpu_decrementer();	1000	start_cpu_decrementer();
965		1001
966	/* FIME: Should make unrelatred change to move snapshot_timebase	1002	/* FIME: Should make unrelatred change to move snapshot_timebase
967	* call here ! */	1003	* call here ! */
968	register_decrementer_clockevent(smp_processor_id());	1004	register_decrementer_clockevent(smp_processor_id());
969	}	1005	}
970		1006
971	/* This function is only called on the boot processor */	1007	/* This function is only called on the boot processor */
972	void __init time_init(void)	1008	void __init time_init(void)
973	{	1009	{
974	unsigned long flags;	1010	unsigned long flags;
975	struct div_result res;	1011	struct div_result res;
976	u64 scale, x;	1012	u64 scale, x;
977	unsigned shift;	1013	unsigned shift;
978		1014
979	if (__USE_RTC()) {	1015	if (__USE_RTC()) {
980	/* 601 processor: dec counts down by 128 every 128ns */	1016	/* 601 processor: dec counts down by 128 every 128ns */
981	ppc_tb_freq = 1000000000;	1017	ppc_tb_freq = 1000000000;
982	tb_last_jiffy = get_rtcl();	1018	tb_last_jiffy = get_rtcl();
983	} else {	1019	} else {
984	/* Normal PowerPC with timebase register */	1020	/* Normal PowerPC with timebase register */
985	ppc_md.calibrate_decr();	1021	ppc_md.calibrate_decr();
986	printk(KERN_DEBUG "time_init: decrementer frequency = %lu.%.6lu MHz\n",	1022	printk(KERN_DEBUG "time_init: decrementer frequency = %lu.%.6lu MHz\n",
987	ppc_tb_freq / 1000000, ppc_tb_freq % 1000000);	1023	ppc_tb_freq / 1000000, ppc_tb_freq % 1000000);
988	printk(KERN_DEBUG "time_init: processor frequency = %lu.%.6lu MHz\n",	1024	printk(KERN_DEBUG "time_init: processor frequency = %lu.%.6lu MHz\n",
989	ppc_proc_freq / 1000000, ppc_proc_freq % 1000000);	1025	ppc_proc_freq / 1000000, ppc_proc_freq % 1000000);
990	tb_last_jiffy = get_tb();	1026	tb_last_jiffy = get_tb();
991	}	1027	}
992		1028
993	tb_ticks_per_jiffy = ppc_tb_freq / HZ;	1029	tb_ticks_per_jiffy = ppc_tb_freq / HZ;
994	tb_ticks_per_sec = ppc_tb_freq;	1030	tb_ticks_per_sec = ppc_tb_freq;
995	tb_ticks_per_usec = ppc_tb_freq / 1000000;	1031	tb_ticks_per_usec = ppc_tb_freq / 1000000;
996	tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000);	1032	tb_to_us = mulhwu_scale_factor(ppc_tb_freq, 1000000);
997	calc_cputime_factors();	1033	calc_cputime_factors();
998	setup_cputime_one_jiffy();	1034	setup_cputime_one_jiffy();
999		1035
1000	/*	1036	/*
1001	* Calculate the length of each tick in ns. It will not be	1037	* Calculate the length of each tick in ns. It will not be
1002	* exactly 1e9/HZ unless ppc_tb_freq is divisible by HZ.	1038	* exactly 1e9/HZ unless ppc_tb_freq is divisible by HZ.
1003	* We compute 1e9 * tb_ticks_per_jiffy / ppc_tb_freq,	1039	* We compute 1e9 * tb_ticks_per_jiffy / ppc_tb_freq,
1004	* rounded up.	1040	* rounded up.
1005	*/	1041	*/
1006	x = (u64) NSEC_PER_SEC * tb_ticks_per_jiffy + ppc_tb_freq - 1;	1042	x = (u64) NSEC_PER_SEC * tb_ticks_per_jiffy + ppc_tb_freq - 1;
1007	do_div(x, ppc_tb_freq);	1043	do_div(x, ppc_tb_freq);
1008	tick_nsec = x;	1044	tick_nsec = x;
1009	last_tick_len = x << TICKLEN_SCALE;	1045	last_tick_len = x << TICKLEN_SCALE;
1010		1046
1011	/*	1047	/*
1012	* Compute ticklen_to_xs, which is a factor which gets multiplied	1048	* Compute ticklen_to_xs, which is a factor which gets multiplied
1013	* by (last_tick_len << TICKLEN_SHIFT) to get a tb_to_xs value.	1049	* by (last_tick_len << TICKLEN_SHIFT) to get a tb_to_xs value.
1014	* It is computed as:	1050	* It is computed as:
1015	* ticklen_to_xs = 2^N / (tb_ticks_per_jiffy * 1e9)	1051	* ticklen_to_xs = 2^N / (tb_ticks_per_jiffy * 1e9)
1016	* where N = 64 + 20 - TICKLEN_SCALE - TICKLEN_SHIFT	1052	* where N = 64 + 20 - TICKLEN_SCALE - TICKLEN_SHIFT
1017	* which turns out to be N = 51 - SHIFT_HZ.	1053	* which turns out to be N = 51 - SHIFT_HZ.
1018	* This gives the result as a 0.64 fixed-point fraction.	1054	* This gives the result as a 0.64 fixed-point fraction.
1019	* That value is reduced by an offset amounting to 1 xsec per	1055	* That value is reduced by an offset amounting to 1 xsec per
1020	* 2^31 timebase ticks to avoid problems with time going backwards	1056	* 2^31 timebase ticks to avoid problems with time going backwards
1021	* by 1 xsec when we do timer_recalc_offset due to losing the	1057	* by 1 xsec when we do timer_recalc_offset due to losing the
1022	* fractional xsec. That offset is equal to ppc_tb_freq/2^51	1058	* fractional xsec. That offset is equal to ppc_tb_freq/2^51
1023	* since there are 2^20 xsec in a second.	1059	* since there are 2^20 xsec in a second.
1024	*/	1060	*/
1025	div128_by_32((1ULL << 51) - ppc_tb_freq, 0,	1061	div128_by_32((1ULL << 51) - ppc_tb_freq, 0,
1026	tb_ticks_per_jiffy << SHIFT_HZ, &res);	1062	tb_ticks_per_jiffy << SHIFT_HZ, &res);
1027	div128_by_32(res.result_high, res.result_low, NSEC_PER_SEC, &res);	1063	div128_by_32(res.result_high, res.result_low, NSEC_PER_SEC, &res);
1028	ticklen_to_xs = res.result_low;	1064	ticklen_to_xs = res.result_low;
1029		1065
1030	/* Compute tb_to_xs from tick_nsec */	1066	/* Compute tb_to_xs from tick_nsec */
1031	tb_to_xs = mulhdu(last_tick_len << TICKLEN_SHIFT, ticklen_to_xs);	1067	tb_to_xs = mulhdu(last_tick_len << TICKLEN_SHIFT, ticklen_to_xs);
1032		1068
1033	/*	1069	/*
1034	* Compute scale factor for sched_clock.	1070	* Compute scale factor for sched_clock.
1035	* The calibrate_decr() function has set tb_ticks_per_sec,	1071	* The calibrate_decr() function has set tb_ticks_per_sec,
1036	* which is the timebase frequency.	1072	* which is the timebase frequency.
1037	* We compute 1e9 * 2^64 / tb_ticks_per_sec and interpret	1073	* We compute 1e9 * 2^64 / tb_ticks_per_sec and interpret
1038	* the 128-bit result as a 64.64 fixed-point number.	1074	* the 128-bit result as a 64.64 fixed-point number.
1039	* We then shift that number right until it is less than 1.0,	1075	* We then shift that number right until it is less than 1.0,
1040	* giving us the scale factor and shift count to use in	1076	* giving us the scale factor and shift count to use in
1041	* sched_clock().	1077	* sched_clock().
1042	*/	1078	*/
1043	div128_by_32(1000000000, 0, tb_ticks_per_sec, &res);	1079	div128_by_32(1000000000, 0, tb_ticks_per_sec, &res);
1044	scale = res.result_low;	1080	scale = res.result_low;
1045	for (shift = 0; res.result_high != 0; ++shift) {	1081	for (shift = 0; res.result_high != 0; ++shift) {
1046	scale = (scale >> 1) \| (res.result_high << 63);	1082	scale = (scale >> 1) \| (res.result_high << 63);
1047	res.result_high >>= 1;	1083	res.result_high >>= 1;
1048	}	1084	}
1049	tb_to_ns_scale = scale;	1085	tb_to_ns_scale = scale;
1050	tb_to_ns_shift = shift;	1086	tb_to_ns_shift = shift;
1051	/* Save the current timebase to pretty up CONFIG_PRINTK_TIME */	1087	/* Save the current timebase to pretty up CONFIG_PRINTK_TIME */
1052	boot_tb = get_tb_or_rtc();	1088	boot_tb = get_tb_or_rtc();
1053		1089
1054	write_seqlock_irqsave(&xtime_lock, flags);	1090	write_seqlock_irqsave(&xtime_lock, flags);
1055		1091
1056	/* If platform provided a timezone (pmac), we correct the time */	1092	/* If platform provided a timezone (pmac), we correct the time */
1057	if (timezone_offset) {	1093	if (timezone_offset) {
1058	sys_tz.tz_minuteswest = -timezone_offset / 60;	1094	sys_tz.tz_minuteswest = -timezone_offset / 60;
1059	sys_tz.tz_dsttime = 0;	1095	sys_tz.tz_dsttime = 0;
1060	}	1096	}
1061		1097
1062	vdso_data->tb_orig_stamp = tb_last_jiffy;	1098	vdso_data->tb_orig_stamp = tb_last_jiffy;
1063	vdso_data->tb_update_count = 0;	1099	vdso_data->tb_update_count = 0;
1064	vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;	1100	vdso_data->tb_ticks_per_sec = tb_ticks_per_sec;
1065	vdso_data->stamp_xsec = (u64) xtime.tv_sec * XSEC_PER_SEC;	1101	vdso_data->stamp_xsec = (u64) xtime.tv_sec * XSEC_PER_SEC;
1066	vdso_data->tb_to_xs = tb_to_xs;	1102	vdso_data->tb_to_xs = tb_to_xs;
1067		1103
1068	write_sequnlock_irqrestore(&xtime_lock, flags);	1104	write_sequnlock_irqrestore(&xtime_lock, flags);
1069		1105
1070	/* Start the decrementer on CPUs that have manual control	1106	/* Start the decrementer on CPUs that have manual control
1071	* such as BookE	1107	* such as BookE
1072	*/	1108	*/
1073	start_cpu_decrementer();	1109	start_cpu_decrementer();
1074		1110
1075	/* Register the clocksource, if we're not running on iSeries */	1111	/* Register the clocksource, if we're not running on iSeries */
1076	if (!firmware_has_feature(FW_FEATURE_ISERIES))	1112	if (!firmware_has_feature(FW_FEATURE_ISERIES))
1077	clocksource_init();	1113	clocksource_init();
1078		1114
1079	init_decrementer_clockevent();	1115	init_decrementer_clockevent();
1080	}	1116	}
1081		1117
1082		1118
1083	#define FEBRUARY 2	1119	#define FEBRUARY 2
1084	#define STARTOFTIME 1970	1120	#define STARTOFTIME 1970
1085	#define SECDAY 86400L	1121	#define SECDAY 86400L
1086	#define SECYR (SECDAY * 365)	1122	#define SECYR (SECDAY * 365)
1087	#define leapyear(year) ((year) % 4 == 0 && \	1123	#define leapyear(year) ((year) % 4 == 0 && \
1088	((year) % 100 != 0 \|\| (year) % 400 == 0))	1124	((year) % 100 != 0 \|\| (year) % 400 == 0))
1089	#define days_in_year(a) (leapyear(a) ? 366 : 365)	1125	#define days_in_year(a) (leapyear(a) ? 366 : 365)
1090	#define days_in_month(a) (month_days[(a) - 1])	1126	#define days_in_month(a) (month_days[(a) - 1])
1091		1127
1092	static int month_days[12] = {	1128	static int month_days[12] = {
1093	31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31	1129	31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
1094	};	1130	};
1095		1131
1096	/*	1132	/*
1097	* This only works for the Gregorian calendar - i.e. after 1752 (in the UK)	1133	* This only works for the Gregorian calendar - i.e. after 1752 (in the UK)
1098	*/	1134	*/
1099	void GregorianDay(struct rtc_time * tm)	1135	void GregorianDay(struct rtc_time * tm)
1100	{	1136	{
1101	int leapsToDate;	1137	int leapsToDate;
1102	int lastYear;	1138	int lastYear;
1103	int day;	1139	int day;
1104	int MonthOffset[] = { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 };	1140	int MonthOffset[] = { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 };
1105		1141
1106	lastYear = tm->tm_year - 1;	1142	lastYear = tm->tm_year - 1;
1107		1143
1108	/*	1144	/*
1109	* Number of leap corrections to apply up to end of last year	1145	* Number of leap corrections to apply up to end of last year
1110	*/	1146	*/
1111	leapsToDate = lastYear / 4 - lastYear / 100 + lastYear / 400;	1147	leapsToDate = lastYear / 4 - lastYear / 100 + lastYear / 400;
1112		1148
1113	/*	1149	/*
1114	* This year is a leap year if it is divisible by 4 except when it is	1150	* This year is a leap year if it is divisible by 4 except when it is
1115	* divisible by 100 unless it is divisible by 400	1151	* divisible by 100 unless it is divisible by 400
1116	*	1152	*
1117	* e.g. 1904 was a leap year, 1900 was not, 1996 is, and 2000 was	1153	* e.g. 1904 was a leap year, 1900 was not, 1996 is, and 2000 was
1118	*/	1154	*/
1119	day = tm->tm_mon > 2 && leapyear(tm->tm_year);	1155	day = tm->tm_mon > 2 && leapyear(tm->tm_year);
1120		1156
1121	day += lastYear*365 + leapsToDate + MonthOffset[tm->tm_mon-1] +	1157	day += lastYear*365 + leapsToDate + MonthOffset[tm->tm_mon-1] +
1122	tm->tm_mday;	1158	tm->tm_mday;
1123		1159
1124	tm->tm_wday = day % 7;	1160	tm->tm_wday = day % 7;
1125	}	1161	}
1126		1162
1127	void to_tm(int tim, struct rtc_time * tm)	1163	void to_tm(int tim, struct rtc_time * tm)
1128	{	1164	{
1129	register int i;	1165	register int i;
1130	register long hms, day;	1166	register long hms, day;
1131		1167
1132	day = tim / SECDAY;	1168	day = tim / SECDAY;
1133	hms = tim % SECDAY;	1169	hms = tim % SECDAY;
1134		1170
1135	/* Hours, minutes, seconds are easy */	1171	/* Hours, minutes, seconds are easy */
1136	tm->tm_hour = hms / 3600;	1172	tm->tm_hour = hms / 3600;
1137	tm->tm_min = (hms % 3600) / 60;	1173	tm->tm_min = (hms % 3600) / 60;
1138	tm->tm_sec = (hms % 3600) % 60;	1174	tm->tm_sec = (hms % 3600) % 60;
1139		1175
1140	/* Number of years in days */	1176	/* Number of years in days */
1141	for (i = STARTOFTIME; day >= days_in_year(i); i++)	1177	for (i = STARTOFTIME; day >= days_in_year(i); i++)
1142	day -= days_in_year(i);	1178	day -= days_in_year(i);
1143	tm->tm_year = i;	1179	tm->tm_year = i;
1144		1180
1145	/* Number of months in days left */	1181	/* Number of months in days left */
1146	if (leapyear(tm->tm_year))	1182	if (leapyear(tm->tm_year))
1147	days_in_month(FEBRUARY) = 29;	1183	days_in_month(FEBRUARY) = 29;
1148	for (i = 1; day >= days_in_month(i); i++)	1184	for (i = 1; day >= days_in_month(i); i++)
1149	day -= days_in_month(i);	1185	day -= days_in_month(i);
1150	days_in_month(FEBRUARY) = 28;	1186	days_in_month(FEBRUARY) = 28;
1151	tm->tm_mon = i;	1187	tm->tm_mon = i;
1152		1188
1153	/* Days are what is left over (+1) from all that. */	1189	/* Days are what is left over (+1) from all that. */
1154	tm->tm_mday = day + 1;	1190	tm->tm_mday = day + 1;
1155		1191
1156	/*	1192	/*
1157	* Determine the day of week	1193	* Determine the day of week
1158	*/	1194	*/
1159	GregorianDay(tm);	1195	GregorianDay(tm);
1160	}	1196	}
1161		1197
1162	/* Auxiliary function to compute scaling factors */	1198	/* Auxiliary function to compute scaling factors */
1163	/* Actually the choice of a timebase running at 1/4 the of the bus	1199	/* Actually the choice of a timebase running at 1/4 the of the bus
1164	* frequency giving resolution of a few tens of nanoseconds is quite nice.	1200	* frequency giving resolution of a few tens of nanoseconds is quite nice.
1165	* It makes this computation very precise (27-28 bits typically) which	1201	* It makes this computation very precise (27-28 bits typically) which
1166	* is optimistic considering the stability of most processor clock	1202	* is optimistic considering the stability of most processor clock
1167	* oscillators and the precision with which the timebase frequency	1203	* oscillators and the precision with which the timebase frequency
1168	* is measured but does not harm.	1204	* is measured but does not harm.
1169	*/	1205	*/
1170	unsigned mulhwu_scale_factor(unsigned inscale, unsigned outscale)	1206	unsigned mulhwu_scale_factor(unsigned inscale, unsigned outscale)
1171	{	1207	{
1172	unsigned mlt=0, tmp, err;	1208	unsigned mlt=0, tmp, err;
1173	/* No concern for performance, it's done once: use a stupid	1209	/* No concern for performance, it's done once: use a stupid
1174	* but safe and compact method to find the multiplier.	1210	* but safe and compact method to find the multiplier.
1175	*/	1211	*/
1176		1212
1177	for (tmp = 1U<<31; tmp != 0; tmp >>= 1) {	1213	for (tmp = 1U<<31; tmp != 0; tmp >>= 1) {
1178	if (mulhwu(inscale, mlt\|tmp) < outscale)	1214	if (mulhwu(inscale, mlt\|tmp) < outscale)
1179	mlt \|= tmp;	1215	mlt \|= tmp;
1180	}	1216	}
1181		1217
1182	/* We might still be off by 1 for the best approximation.	1218	/* We might still be off by 1 for the best approximation.
1183	* A side effect of this is that if outscale is too large	1219	* A side effect of this is that if outscale is too large
1184	* the returned value will be zero.	1220	* the returned value will be zero.
1185	* Many corner cases have been checked and seem to work,	1221	* Many corner cases have been checked and seem to work,
1186	* some might have been forgotten in the test however.	1222	* some might have been forgotten in the test however.
1187	*/	1223	*/
1188		1224
1189	err = inscale * (mlt+1);	1225	err = inscale * (mlt+1);
1190	if (err <= inscale/2)	1226	if (err <= inscale/2)
1191	mlt++;	1227	mlt++;
1192	return mlt;	1228	return mlt;
1193	}	1229	}
1194		1230
1195	/*	1231	/*
1196	* Divide a 128-bit dividend by a 32-bit divisor, leaving a 128 bit	1232	* Divide a 128-bit dividend by a 32-bit divisor, leaving a 128 bit
1197	* result.	1233	* result.
1198	*/	1234	*/
1199	void div128_by_32(u64 dividend_high, u64 dividend_low,	1235	void div128_by_32(u64 dividend_high, u64 dividend_low,
1200	unsigned divisor, struct div_result *dr)	1236	unsigned divisor, struct div_result *dr)
1201	{	1237	{
1202	unsigned long a, b, c, d;	1238	unsigned long a, b, c, d;
1203	unsigned long w, x, y, z;	1239	unsigned long w, x, y, z;
1204	u64 ra, rb, rc;	1240	u64 ra, rb, rc;
1205		1241
1206	a = dividend_high >> 32;	1242	a = dividend_high >> 32;
1207	b = dividend_high & 0xffffffff;	1243	b = dividend_high & 0xffffffff;
1208	c = dividend_low >> 32;	1244	c = dividend_low >> 32;
1209	d = dividend_low & 0xffffffff;	1245	d = dividend_low & 0xffffffff;
1210		1246
1211	w = a / divisor;	1247	w = a / divisor;
1212	ra = ((u64)(a - (w * divisor)) << 32) + b;	1248	ra = ((u64)(a - (w * divisor)) << 32) + b;
1213		1249
1214	rb = ((u64) do_div(ra, divisor) << 32) + c;	1250	rb = ((u64) do_div(ra, divisor) << 32) + c;
1215	x = ra;	1251	x = ra;
1216		1252
1217	rc = ((u64) do_div(rb, divisor) << 32) + d;	1253	rc = ((u64) do_div(rb, divisor) << 32) + d;
1218	y = rb;	1254	y = rb;
1219		1255
1220	do_div(rc, divisor);	1256	do_div(rc, divisor);
1221	z = rc;	1257	z = rc;
1222		1258
1223	dr->result_high = ((u64)w << 32) + x;	1259	dr->result_high = ((u64)w << 32) + x;
1224	dr->result_low = ((u64)y << 32) + z;	1260	dr->result_low = ((u64)y << 32) + z;
1225		1261
1226	}	1262	}
1227		1263
1228	/* We don't need to calibrate delay, we use the CPU timebase for that */	1264	/* We don't need to calibrate delay, we use the CPU timebase for that */
1229	void calibrate_delay(void)	1265	void calibrate_delay(void)
1230	{	1266	{
1231	/* Some generic code (such as spinlock debug) use loops_per_jiffy	1267	/* Some generic code (such as spinlock debug) use loops_per_jiffy
1232	* as the number of __delay(1) in a jiffy, so make it so	1268	* as the number of __delay(1) in a jiffy, so make it so
1233	*/	1269	*/
1234	loops_per_jiffy = tb_ticks_per_jiffy;	1270	loops_per_jiffy = tb_ticks_per_jiffy;
1235	}	1271	}
1236		1272
1237	static int __init rtc_init(void)	1273	static int __init rtc_init(void)
1238	{	1274	{
1239	struct platform_device *pdev;	1275	struct platform_device *pdev;
1240		1276
1241	if (!ppc_md.get_rtc_time)	1277	if (!ppc_md.get_rtc_time)
1242	return -ENODEV;	1278	return -ENODEV;
1243		1279
1244	pdev = platform_device_register_simple("rtc-generic", -1, NULL, 0);	1280	pdev = platform_device_register_simple("rtc-generic", -1, NULL, 0);
1245	if (IS_ERR(pdev))	1281	if (IS_ERR(pdev))
1246	return PTR_ERR(pdev);	1282	return PTR_ERR(pdev);
1247		1283