/*
   * linux/percpu-defs.h - basic definitions for percpu areas
   *
   * DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.
   *
   * This file is separate from linux/percpu.h to avoid cyclic inclusion
   * dependency from arch header files.  Only to be included from
   * asm/percpu.h.
   *
   * This file includes macros necessary to declare percpu sections and
   * variables, and definitions of percpu accessors and operations.  It
   * should provide enough percpu features to arch header files even when
   * they can only include asm/percpu.h to avoid cyclic inclusion dependency.
   */

  #ifndef _LINUX_PERCPU_DEFS_H
  #define _LINUX_PERCPU_DEFS_H

  #ifdef CONFIG_SMP
  
  #ifdef MODULE
  #define PER_CPU_SHARED_ALIGNED_SECTION ""
  #define PER_CPU_ALIGNED_SECTION ""
  #else
  #define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"
  #define PER_CPU_ALIGNED_SECTION "..shared_aligned"
  #endif
  #define PER_CPU_FIRST_SECTION "..first"
  
  #else
  
  #define PER_CPU_SHARED_ALIGNED_SECTION ""
  #define PER_CPU_ALIGNED_SECTION "..shared_aligned"
  #define PER_CPU_FIRST_SECTION ""
  
  #endif

  /*

   * Base implementations of per-CPU variable declarations and definitions, where
   * the section in which the variable is to be placed is provided by the

   * 'sec' argument.  This may be used to affect the parameters governing the

   * variable's storage.
   *
   * NOTE!  The sections for the DECLARE and for the DEFINE must match, lest
   * linkage errors occur due the compiler generating the wrong code to access
   * that section.
   */

  #define __PCPU_ATTRS(sec)						\

  	__percpu __attribute__((section(PER_CPU_BASE_SECTION sec)))	\

  	PER_CPU_ATTRIBUTES
  
  #define __PCPU_DUMMY_ATTRS						\
  	__attribute__((section(".discard"), unused))
  
  /*
   * s390 and alpha modules require percpu variables to be defined as
   * weak to force the compiler to generate GOT based external
   * references for them.  This is necessary because percpu sections
   * will be located outside of the usually addressable area.
   *
   * This definition puts the following two extra restrictions when
   * defining percpu variables.
   *
   * 1. The symbol must be globally unique, even the static ones.
   * 2. Static percpu variables cannot be defined inside a function.
   *
   * Archs which need weak percpu definitions should define
   * ARCH_NEEDS_WEAK_PER_CPU in asm/percpu.h when necessary.
   *
   * To ensure that the generic code observes the above two
   * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak
   * definition is used for all cases.
   */
  #if defined(ARCH_NEEDS_WEAK_PER_CPU) || defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)
  /*
   * __pcpu_scope_* dummy variable is used to enforce scope.  It
   * receives the static modifier when it's used in front of
   * DEFINE_PER_CPU() and will trigger build failure if
   * DECLARE_PER_CPU() is used for the same variable.
   *
   * __pcpu_unique_* dummy variable is used to enforce symbol uniqueness
   * such that hidden weak symbol collision, which will cause unrelated
   * variables to share the same address, can be detected during build.
   */
  #define DECLARE_PER_CPU_SECTION(type, name, sec)			\
  	extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name;		\

  	extern __PCPU_ATTRS(sec) __typeof__(type) name

  
  #define DEFINE_PER_CPU_SECTION(type, name, sec)				\
  	__PCPU_DUMMY_ATTRS char __pcpu_scope_##name;			\

  	extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;		\

  	__PCPU_DUMMY_ATTRS char __pcpu_unique_##name;			\

  	extern __PCPU_ATTRS(sec) __typeof__(type) name;			\

  	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak			\

  	__typeof__(type) name

  #else
  /*
   * Normal declaration and definition macros.
   */
  #define DECLARE_PER_CPU_SECTION(type, name, sec)			\

  	extern __PCPU_ATTRS(sec) __typeof__(type) name

  
  #define DEFINE_PER_CPU_SECTION(type, name, sec)				\

  	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES			\

  	__typeof__(type) name

  #endif

  
  /*
   * Variant on the per-CPU variable declaration/definition theme used for
   * ordinary per-CPU variables.
   */
  #define DECLARE_PER_CPU(type, name)					\
  	DECLARE_PER_CPU_SECTION(type, name, "")
  
  #define DEFINE_PER_CPU(type, name)					\
  	DEFINE_PER_CPU_SECTION(type, name, "")
  
  /*
   * Declaration/definition used for per-CPU variables that must come first in
   * the set of variables.
   */
  #define DECLARE_PER_CPU_FIRST(type, name)				\
  	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
  
  #define DEFINE_PER_CPU_FIRST(type, name)				\
  	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)
  
  /*
   * Declaration/definition used for per-CPU variables that must be cacheline
   * aligned under SMP conditions so that, whilst a particular instance of the
   * data corresponds to a particular CPU, inefficiencies due to direct access by
   * other CPUs are reduced by preventing the data from unnecessarily spanning
   * cachelines.
   *
   * An example of this would be statistical data, where each CPU's set of data
   * is updated by that CPU alone, but the data from across all CPUs is collated
   * by a CPU processing a read from a proc file.
   */
  #define DECLARE_PER_CPU_SHARED_ALIGNED(type, name)			\
  	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
  	____cacheline_aligned_in_smp
  
  #define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)			\
  	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \
  	____cacheline_aligned_in_smp

  #define DECLARE_PER_CPU_ALIGNED(type, name)				\
  	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION)	\
  	____cacheline_aligned
  
  #define DEFINE_PER_CPU_ALIGNED(type, name)				\
  	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION)	\
  	____cacheline_aligned

  /*
   * Declaration/definition used for per-CPU variables that must be page aligned.
   */

  #define DECLARE_PER_CPU_PAGE_ALIGNED(type, name)			\

  	DECLARE_PER_CPU_SECTION(type, name, "..page_aligned")		\

  	__aligned(PAGE_SIZE)

  
  #define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)				\

  	DEFINE_PER_CPU_SECTION(type, name, "..page_aligned")		\

  	__aligned(PAGE_SIZE)

  
  /*

   * Declaration/definition used for per-CPU variables that must be read mostly.
   */
  #define DECLARE_PER_CPU_READ_MOSTLY(type, name)			\

  	DECLARE_PER_CPU_SECTION(type, name, "..read_mostly")

  
  #define DEFINE_PER_CPU_READ_MOSTLY(type, name)				\

  	DEFINE_PER_CPU_SECTION(type, name, "..read_mostly")

  
  /*

   * Intermodule exports for per-CPU variables.  sparse forgets about
   * address space across EXPORT_SYMBOL(), change EXPORT_SYMBOL() to
   * noop if __CHECKER__.

   */

  #ifndef __CHECKER__

  #define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var)
  #define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var)

  #else
  #define EXPORT_PER_CPU_SYMBOL(var)
  #define EXPORT_PER_CPU_SYMBOL_GPL(var)
  #endif

  /*
   * Accessors and operations.
   */
  #ifndef __ASSEMBLY__

  /*

   * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating
   * @ptr and is invoked once before a percpu area is accessed by all
   * accessors and operations.  This is performed in the generic part of
   * percpu and arch overrides don't need to worry about it; however, if an
   * arch wants to implement an arch-specific percpu accessor or operation,
   * it may use __verify_pcpu_ptr() to verify the parameters.

   *
   * + 0 is required in order to convert the pointer type from a
   * potential array type to a pointer to a single item of the array.
   */

  #define __verify_pcpu_ptr(ptr)						\
  do {									\

  	const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL;	\
  	(void)__vpp_verify;						\
  } while (0)

  #ifdef CONFIG_SMP
  
  /*
   * Add an offset to a pointer but keep the pointer as-is.  Use RELOC_HIDE()
   * to prevent the compiler from making incorrect assumptions about the
   * pointer value.  The weird cast keeps both GCC and sparse happy.
   */

  #define SHIFT_PERCPU_PTR(__p, __offset)					\

  	RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
  
  #define per_cpu_ptr(ptr, cpu)						\

  ({									\

  	__verify_pcpu_ptr(ptr);						\
  	SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)));			\

  })

  #define raw_cpu_ptr(ptr)						\
  ({									\
  	__verify_pcpu_ptr(ptr);						\
  	arch_raw_cpu_ptr(ptr);						\
  })

  
  #ifdef CONFIG_DEBUG_PREEMPT

  #define this_cpu_ptr(ptr)						\
  ({									\
  	__verify_pcpu_ptr(ptr);						\
  	SHIFT_PERCPU_PTR(ptr, my_cpu_offset);				\
  })

  #else
  #define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
  #endif

  #else	/* CONFIG_SMP */

  #define VERIFY_PERCPU_PTR(__p)						\
  ({									\
  	__verify_pcpu_ptr(__p);						\
  	(typeof(*(__p)) __kernel __force *)(__p);			\

  })

  #define per_cpu_ptr(ptr, cpu)	({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })

  #define raw_cpu_ptr(ptr)	per_cpu_ptr(ptr, 0)
  #define this_cpu_ptr(ptr)	raw_cpu_ptr(ptr)

  
  #endif	/* CONFIG_SMP */

  #define per_cpu(var, cpu)	(*per_cpu_ptr(&(var), cpu))

  /*
   * Must be an lvalue. Since @var must be a simple identifier,
   * we force a syntax error here if it isn't.
   */

  #define get_cpu_var(var)						\
  (*({									\
  	preempt_disable();						\
  	this_cpu_ptr(&var);						\
  }))

  
  /*
   * The weird & is necessary because sparse considers (void)(var) to be
   * a direct dereference of percpu variable (var).
   */

  #define put_cpu_var(var)						\
  do {									\
  	(void)&(var);							\
  	preempt_enable();						\

  } while (0)

  #define get_cpu_ptr(var)						\
  ({									\
  	preempt_disable();						\
  	this_cpu_ptr(var);						\
  })

  #define put_cpu_ptr(var)						\
  do {									\
  	(void)(var);							\
  	preempt_enable();						\

  } while (0)

  /*
   * Branching function to split up a function into a set of functions that
   * are called for different scalar sizes of the objects handled.
   */
  
  extern void __bad_size_call_parameter(void);
  
  #ifdef CONFIG_DEBUG_PREEMPT
  extern void __this_cpu_preempt_check(const char *op);
  #else
  static inline void __this_cpu_preempt_check(const char *op) { }
  #endif
  
  #define __pcpu_size_call_return(stem, variable)				\

  ({									\
  	typeof(variable) pscr_ret__;					\

  	__verify_pcpu_ptr(&(variable));					\
  	switch(sizeof(variable)) {					\

  	case 1: pscr_ret__ = stem##1(variable); break;			\
  	case 2: pscr_ret__ = stem##2(variable); break;			\
  	case 4: pscr_ret__ = stem##4(variable); break;			\
  	case 8: pscr_ret__ = stem##8(variable); break;			\

  	default:							\

  		__bad_size_call_parameter(); break;			\

  	}								\
  	pscr_ret__;							\
  })
  
  #define __pcpu_size_call_return2(stem, variable, ...)			\
  ({									\
  	typeof(variable) pscr2_ret__;					\
  	__verify_pcpu_ptr(&(variable));					\
  	switch(sizeof(variable)) {					\
  	case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break;	\
  	case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break;	\
  	case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break;	\
  	case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break;	\
  	default:							\
  		__bad_size_call_parameter(); break;			\
  	}								\
  	pscr2_ret__;							\
  })
  
  /*
   * Special handling for cmpxchg_double.  cmpxchg_double is passed two
   * percpu variables.  The first has to be aligned to a double word
   * boundary and the second has to follow directly thereafter.
   * We enforce this on all architectures even if they don't support
   * a double cmpxchg instruction, since it's a cheap requirement, and it
   * avoids breaking the requirement for architectures with the instruction.
   */
  #define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...)		\
  ({									\
  	bool pdcrb_ret__;						\

  	__verify_pcpu_ptr(&(pcp1));					\

  	BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2));			\

  	VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1)));	\
  	VM_BUG_ON((unsigned long)(&(pcp2)) !=				\
  		  (unsigned long)(&(pcp1)) + sizeof(pcp1));		\

  	switch(sizeof(pcp1)) {						\
  	case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break;	\
  	case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break;	\
  	case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break;	\
  	case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break;	\
  	default:							\
  		__bad_size_call_parameter(); break;			\
  	}								\
  	pdcrb_ret__;							\
  })
  
  #define __pcpu_size_call(stem, variable, ...)				\
  do {									\
  	__verify_pcpu_ptr(&(variable));					\
  	switch(sizeof(variable)) {					\
  		case 1: stem##1(variable, __VA_ARGS__);break;		\
  		case 2: stem##2(variable, __VA_ARGS__);break;		\
  		case 4: stem##4(variable, __VA_ARGS__);break;		\
  		case 8: stem##8(variable, __VA_ARGS__);break;		\
  		default: 						\
  			__bad_size_call_parameter();break;		\
  	}								\
  } while (0)
  
  /*
   * this_cpu operations (C) 2008-2013 Christoph Lameter <cl@linux.com>
   *
   * Optimized manipulation for memory allocated through the per cpu
   * allocator or for addresses of per cpu variables.
   *
   * These operation guarantee exclusivity of access for other operations
   * on the *same* processor. The assumption is that per cpu data is only
   * accessed by a single processor instance (the current one).
   *
   * The arch code can provide optimized implementation by defining macros
   * for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per
   * cpu atomic operations for 2 byte sized RMW actions. If arch code does
   * not provide operations for a scalar size then the fallback in the
   * generic code will be used.

   *
   * cmpxchg_double replaces two adjacent scalars at once.  The first two
   * parameters are per cpu variables which have to be of the same size.  A
   * truth value is returned to indicate success or failure (since a double
   * register result is difficult to handle).  There is very limited hardware
   * support for these operations, so only certain sizes may work.

   */
  
  /*

   * Operations for contexts where we do not want to do any checks for
   * preemptions.  Unless strictly necessary, always use [__]this_cpu_*()
   * instead.

   *

   * If there is no other protection through preempt disable and/or disabling
   * interupts then one of these RMW operations can show unexpected behavior
   * because the execution thread was rescheduled on another processor or an
   * interrupt occurred and the same percpu variable was modified from the
   * interrupt context.

   */

  #define raw_cpu_read(pcp)		__pcpu_size_call_return(raw_cpu_read_, pcp)
  #define raw_cpu_write(pcp, val)		__pcpu_size_call(raw_cpu_write_, pcp, val)
  #define raw_cpu_add(pcp, val)		__pcpu_size_call(raw_cpu_add_, pcp, val)
  #define raw_cpu_and(pcp, val)		__pcpu_size_call(raw_cpu_and_, pcp, val)
  #define raw_cpu_or(pcp, val)		__pcpu_size_call(raw_cpu_or_, pcp, val)
  #define raw_cpu_add_return(pcp, val)	__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)
  #define raw_cpu_xchg(pcp, nval)		__pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)
  #define raw_cpu_cmpxchg(pcp, oval, nval) \

  	__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)

  #define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
  	__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
  
  #define raw_cpu_sub(pcp, val)		raw_cpu_add(pcp, -(val))
  #define raw_cpu_inc(pcp)		raw_cpu_add(pcp, 1)
  #define raw_cpu_dec(pcp)		raw_cpu_sub(pcp, 1)
  #define raw_cpu_sub_return(pcp, val)	raw_cpu_add_return(pcp, -(typeof(pcp))(val))
  #define raw_cpu_inc_return(pcp)		raw_cpu_add_return(pcp, 1)
  #define raw_cpu_dec_return(pcp)		raw_cpu_add_return(pcp, -1)

  
  /*

   * Operations for contexts that are safe from preemption/interrupts.  These
   * operations verify that preemption is disabled.

   */

  #define __this_cpu_read(pcp)						\
  ({									\
  	__this_cpu_preempt_check("read");				\
  	raw_cpu_read(pcp);						\
  })

  #define __this_cpu_write(pcp, val)					\
  ({									\
  	__this_cpu_preempt_check("write");				\
  	raw_cpu_write(pcp, val);					\
  })

  #define __this_cpu_add(pcp, val)					\
  ({									\
  	__this_cpu_preempt_check("add");				\

  	raw_cpu_add(pcp, val);						\

  })

  #define __this_cpu_and(pcp, val)					\
  ({									\
  	__this_cpu_preempt_check("and");				\

  	raw_cpu_and(pcp, val);						\

  })

  #define __this_cpu_or(pcp, val)						\
  ({									\
  	__this_cpu_preempt_check("or");					\

  	raw_cpu_or(pcp, val);						\

  })

  #define __this_cpu_add_return(pcp, val)					\
  ({									\
  	__this_cpu_preempt_check("add_return");				\
  	raw_cpu_add_return(pcp, val);					\
  })

  #define __this_cpu_xchg(pcp, nval)					\
  ({									\
  	__this_cpu_preempt_check("xchg");				\
  	raw_cpu_xchg(pcp, nval);					\
  })

  #define __this_cpu_cmpxchg(pcp, oval, nval)				\
  ({									\
  	__this_cpu_preempt_check("cmpxchg");				\
  	raw_cpu_cmpxchg(pcp, oval, nval);				\
  })

  #define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
  ({	__this_cpu_preempt_check("cmpxchg_double");			\
  	raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2);	\
  })

  #define __this_cpu_sub(pcp, val)	__this_cpu_add(pcp, -(typeof(pcp))(val))
  #define __this_cpu_inc(pcp)		__this_cpu_add(pcp, 1)
  #define __this_cpu_dec(pcp)		__this_cpu_sub(pcp, 1)
  #define __this_cpu_sub_return(pcp, val)	__this_cpu_add_return(pcp, -(typeof(pcp))(val))
  #define __this_cpu_inc_return(pcp)	__this_cpu_add_return(pcp, 1)
  #define __this_cpu_dec_return(pcp)	__this_cpu_add_return(pcp, -1)

  
  /*

   * Operations with implied preemption/interrupt protection.  These
   * operations can be used without worrying about preemption or interrupt.

   */

  #define this_cpu_read(pcp)		__pcpu_size_call_return(this_cpu_read_, pcp)
  #define this_cpu_write(pcp, val)	__pcpu_size_call(this_cpu_write_, pcp, val)
  #define this_cpu_add(pcp, val)		__pcpu_size_call(this_cpu_add_, pcp, val)
  #define this_cpu_and(pcp, val)		__pcpu_size_call(this_cpu_and_, pcp, val)
  #define this_cpu_or(pcp, val)		__pcpu_size_call(this_cpu_or_, pcp, val)
  #define this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)
  #define this_cpu_xchg(pcp, nval)	__pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)
  #define this_cpu_cmpxchg(pcp, oval, nval) \
  	__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)
  #define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
  	__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)
  
  #define this_cpu_sub(pcp, val)		this_cpu_add(pcp, -(typeof(pcp))(val))
  #define this_cpu_inc(pcp)		this_cpu_add(pcp, 1)
  #define this_cpu_dec(pcp)		this_cpu_sub(pcp, 1)

  #define this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(typeof(pcp))(val))
  #define this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)
  #define this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)

  #endif /* __ASSEMBLY__ */

  #endif /* _LINUX_PERCPU_DEFS_H */