Subversion Repositories Kolibri OS

/*

 * linux/percpu-defs.h - basic definitions for percpu areas

 *

 * DO NOT INCLUDE DIRECTLY OUTSIDE PERCPU IMPLEMENTATION PROPER.

 *

 * This file is separate from linux/percpu.h to avoid cyclic inclusion

 * dependency from arch header files.  Only to be included from

 * asm/percpu.h.

 *

 * This file includes macros necessary to declare percpu sections and

 * variables, and definitions of percpu accessors and operations.  It

 * should provide enough percpu features to arch header files even when

 * they can only include asm/percpu.h to avoid cyclic inclusion dependency.

 */

#ifndef _LINUX_PERCPU_DEFS_H

#define _LINUX_PERCPU_DEFS_H

#ifdef CONFIG_SMP

#ifdef MODULE

#define PER_CPU_SHARED_ALIGNED_SECTION ""

#define PER_CPU_ALIGNED_SECTION ""

#else

#define PER_CPU_SHARED_ALIGNED_SECTION "..shared_aligned"

#define PER_CPU_ALIGNED_SECTION "..shared_aligned"

#endif

#define PER_CPU_FIRST_SECTION "..first"

#else

#define PER_CPU_SHARED_ALIGNED_SECTION ""

#define PER_CPU_ALIGNED_SECTION "..shared_aligned"

#define PER_CPU_FIRST_SECTION ""

#endif

/*

 * Base implementations of per-CPU variable declarations and definitions, where

 * the section in which the variable is to be placed is provided by the

 * 'sec' argument.  This may be used to affect the parameters governing the

 * variable's storage.

 *

 * NOTE!  The sections for the DECLARE and for the DEFINE must match, lest

 * linkage errors occur due the compiler generating the wrong code to access

 * that section.

 */

#define __PCPU_ATTRS(sec)						\

	__percpu __attribute__((section(PER_CPU_BASE_SECTION sec)))	\

	PER_CPU_ATTRIBUTES

#define __PCPU_DUMMY_ATTRS						\

	__attribute__((section(".discard"), unused))

/*

 * s390 and alpha modules require percpu variables to be defined as

 * weak to force the compiler to generate GOT based external

 * references for them.  This is necessary because percpu sections

 * will be located outside of the usually addressable area.

 *

 * This definition puts the following two extra restrictions when

 * defining percpu variables.

 *

 * 1. The symbol must be globally unique, even the static ones.

 * 2. Static percpu variables cannot be defined inside a function.

 *

 * Archs which need weak percpu definitions should define

 * ARCH_NEEDS_WEAK_PER_CPU in asm/percpu.h when necessary.

 *

 * To ensure that the generic code observes the above two

 * restrictions, if CONFIG_DEBUG_FORCE_WEAK_PER_CPU is set weak

 * definition is used for all cases.

 */

#if defined(ARCH_NEEDS_WEAK_PER_CPU) || defined(CONFIG_DEBUG_FORCE_WEAK_PER_CPU)

/*

 * __pcpu_scope_* dummy variable is used to enforce scope.  It

 * receives the static modifier when it's used in front of

 * DEFINE_PER_CPU() and will trigger build failure if

 * DECLARE_PER_CPU() is used for the same variable.

 *

 * __pcpu_unique_* dummy variable is used to enforce symbol uniqueness

 * such that hidden weak symbol collision, which will cause unrelated

 * variables to share the same address, can be detected during build.

 */

#define DECLARE_PER_CPU_SECTION(type, name, sec)			\

	extern __PCPU_DUMMY_ATTRS char __pcpu_scope_##name;		\

	extern __PCPU_ATTRS(sec) __typeof__(type) name

#define DEFINE_PER_CPU_SECTION(type, name, sec)				\

	__PCPU_DUMMY_ATTRS char __pcpu_scope_##name;			\

	extern __PCPU_DUMMY_ATTRS char __pcpu_unique_##name;		\

	__PCPU_DUMMY_ATTRS char __pcpu_unique_##name;			\

	extern __PCPU_ATTRS(sec) __typeof__(type) name;			\

	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES __weak			\

	__typeof__(type) name

#else

/*

 * Normal declaration and definition macros.

 */

#define DECLARE_PER_CPU_SECTION(type, name, sec)			\

	extern __PCPU_ATTRS(sec) __typeof__(type) name

#define DEFINE_PER_CPU_SECTION(type, name, sec)				\

	__PCPU_ATTRS(sec) PER_CPU_DEF_ATTRIBUTES			\

	__typeof__(type) name

#endif

/*

 * Variant on the per-CPU variable declaration/definition theme used for

 * ordinary per-CPU variables.

 */

#define DECLARE_PER_CPU(type, name)					\

	DECLARE_PER_CPU_SECTION(type, name, "")

#define DEFINE_PER_CPU(type, name)					\

	DEFINE_PER_CPU_SECTION(type, name, "")

/*

 * Declaration/definition used for per-CPU variables that must come first in

 * the set of variables.

 */

#define DECLARE_PER_CPU_FIRST(type, name)				\

	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)

#define DEFINE_PER_CPU_FIRST(type, name)				\

	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_FIRST_SECTION)

/*

 * Declaration/definition used for per-CPU variables that must be cacheline

 * aligned under SMP conditions so that, whilst a particular instance of the

 * data corresponds to a particular CPU, inefficiencies due to direct access by

 * other CPUs are reduced by preventing the data from unnecessarily spanning

 * cachelines.

 *

 * An example of this would be statistical data, where each CPU's set of data

 * is updated by that CPU alone, but the data from across all CPUs is collated

 * by a CPU processing a read from a proc file.

 */

#define DECLARE_PER_CPU_SHARED_ALIGNED(type, name)			\

	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \

	____cacheline_aligned_in_smp

#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name)			\

	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_SHARED_ALIGNED_SECTION) \

	____cacheline_aligned_in_smp

#define DECLARE_PER_CPU_ALIGNED(type, name)				\

	DECLARE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION)	\

	____cacheline_aligned

#define DEFINE_PER_CPU_ALIGNED(type, name)				\

	DEFINE_PER_CPU_SECTION(type, name, PER_CPU_ALIGNED_SECTION)	\

	____cacheline_aligned

/*

 * Declaration/definition used for per-CPU variables that must be page aligned.

 */

#define DECLARE_PER_CPU_PAGE_ALIGNED(type, name)			\

	DECLARE_PER_CPU_SECTION(type, name, "..page_aligned")		\

	__aligned(PAGE_SIZE)

#define DEFINE_PER_CPU_PAGE_ALIGNED(type, name)				\

	DEFINE_PER_CPU_SECTION(type, name, "..page_aligned")		\

	__aligned(PAGE_SIZE)

/*

 * Declaration/definition used for per-CPU variables that must be read mostly.

 */

#define DECLARE_PER_CPU_READ_MOSTLY(type, name)			\

	DECLARE_PER_CPU_SECTION(type, name, "..read_mostly")

#define DEFINE_PER_CPU_READ_MOSTLY(type, name)				\

	DEFINE_PER_CPU_SECTION(type, name, "..read_mostly")

/*

 * Intermodule exports for per-CPU variables.  sparse forgets about

 * address space across EXPORT_SYMBOL(), change EXPORT_SYMBOL() to

 * noop if __CHECKER__.

 */

#ifndef __CHECKER__

#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(var)

#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(var)

#else

#define EXPORT_PER_CPU_SYMBOL(var)

#define EXPORT_PER_CPU_SYMBOL_GPL(var)

#endif

/*

 * Accessors and operations.

 */

#ifndef __ASSEMBLY__

/*

 * __verify_pcpu_ptr() verifies @ptr is a percpu pointer without evaluating

 * @ptr and is invoked once before a percpu area is accessed by all

 * accessors and operations.  This is performed in the generic part of

 * percpu and arch overrides don't need to worry about it; however, if an

 * arch wants to implement an arch-specific percpu accessor or operation,

 * it may use __verify_pcpu_ptr() to verify the parameters.

 *

 * + 0 is required in order to convert the pointer type from a

 * potential array type to a pointer to a single item of the array.

 */

#define __verify_pcpu_ptr(ptr)						\

do {									\

	const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL;	\

	(void)__vpp_verify;						\

} while (0)

#ifdef CONFIG_SMP

/*

 * Add an offset to a pointer but keep the pointer as-is.  Use RELOC_HIDE()

 * to prevent the compiler from making incorrect assumptions about the

 * pointer value.  The weird cast keeps both GCC and sparse happy.

 */

#define SHIFT_PERCPU_PTR(__p, __offset)					\

	RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))

#define per_cpu_ptr(ptr, cpu)						\

({									\

	__verify_pcpu_ptr(ptr);						\

	SHIFT_PERCPU_PTR((ptr), per_cpu_offset((cpu)));			\

})

#define raw_cpu_ptr(ptr)						\

({									\

	__verify_pcpu_ptr(ptr);						\

	arch_raw_cpu_ptr(ptr);						\

})

#ifdef CONFIG_DEBUG_PREEMPT

#define this_cpu_ptr(ptr)						\

({									\

	__verify_pcpu_ptr(ptr);						\

	SHIFT_PERCPU_PTR(ptr, my_cpu_offset);				\

})

#else

#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)

#endif

#else	/* CONFIG_SMP */

#define VERIFY_PERCPU_PTR(__p)						\

({									\

	__verify_pcpu_ptr(__p);						\

	(typeof(*(__p)) __kernel __force *)(__p);			\

})

#define per_cpu_ptr(ptr, cpu)	({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })

#define raw_cpu_ptr(ptr)	per_cpu_ptr(ptr, 0)

#define this_cpu_ptr(ptr)	raw_cpu_ptr(ptr)

#endif	/* CONFIG_SMP */

#define per_cpu(var, cpu)	(*per_cpu_ptr(&(var), cpu))

/*

 * Must be an lvalue. Since @var must be a simple identifier,

 * we force a syntax error here if it isn't.

 */

#define get_cpu_var(var)						\

(*({									\

	preempt_disable();						\

	this_cpu_ptr(&var);						\

}))

/*

 * The weird & is necessary because sparse considers (void)(var) to be

 * a direct dereference of percpu variable (var).

 */

#define put_cpu_var(var)						\

do {									\

	(void)&(var);							\

	preempt_enable();						\

} while (0)

#define get_cpu_ptr(var)						\

({									\

	preempt_disable();						\

	this_cpu_ptr(var);						\

})

#define put_cpu_ptr(var)						\

do {									\

	(void)(var);							\

	preempt_enable();						\

} while (0)

/*

 * Branching function to split up a function into a set of functions that

 * are called for different scalar sizes of the objects handled.

 */

extern void __bad_size_call_parameter(void);

#ifdef CONFIG_DEBUG_PREEMPT

extern void __this_cpu_preempt_check(const char *op);

#else

static inline void __this_cpu_preempt_check(const char *op) { }

#endif

#define __pcpu_size_call_return(stem, variable)				\

({									\

	typeof(variable) pscr_ret__;					\

	__verify_pcpu_ptr(&(variable));					\

	switch(sizeof(variable)) {					\

	case 1: pscr_ret__ = stem##1(variable); break;			\

	case 2: pscr_ret__ = stem##2(variable); break;			\

	case 4: pscr_ret__ = stem##4(variable); break;			\

	case 8: pscr_ret__ = stem##8(variable); break;			\

	default:							\

		__bad_size_call_parameter(); break;			\

	}								\

	pscr_ret__;							\

})

#define __pcpu_size_call_return2(stem, variable, ...)			\

({									\

	typeof(variable) pscr2_ret__;					\

	__verify_pcpu_ptr(&(variable));					\

	switch(sizeof(variable)) {					\

	case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break;	\

	case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break;	\

	case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break;	\

	case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break;	\

	default:							\

		__bad_size_call_parameter(); break;			\

	}								\

	pscr2_ret__;							\

})

/*

 * Special handling for cmpxchg_double.  cmpxchg_double is passed two

 * percpu variables.  The first has to be aligned to a double word

 * boundary and the second has to follow directly thereafter.

 * We enforce this on all architectures even if they don't support

 * a double cmpxchg instruction, since it's a cheap requirement, and it

 * avoids breaking the requirement for architectures with the instruction.

 */

#define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...)		\

({									\

	bool pdcrb_ret__;						\

	__verify_pcpu_ptr(&(pcp1));					\

	BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2));			\

	VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1)));	\

	VM_BUG_ON((unsigned long)(&(pcp2)) !=				\

		  (unsigned long)(&(pcp1)) + sizeof(pcp1));		\

	switch(sizeof(pcp1)) {						\

	case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break;	\

	case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break;	\

	case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break;	\

	case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break;	\

	default:							\

		__bad_size_call_parameter(); break;			\

	}								\

	pdcrb_ret__;							\

})

#define __pcpu_size_call(stem, variable, ...)				\

do {									\

	__verify_pcpu_ptr(&(variable));					\

	switch(sizeof(variable)) {					\

		case 1: stem##1(variable, __VA_ARGS__);break;		\

		case 2: stem##2(variable, __VA_ARGS__);break;		\

		case 4: stem##4(variable, __VA_ARGS__);break;		\

		case 8: stem##8(variable, __VA_ARGS__);break;		\

		default: 						\

			__bad_size_call_parameter();break;		\

	}								\

} while (0)

/*

 * this_cpu operations (C) 2008-2013 Christoph Lameter 

 *

 * Optimized manipulation for memory allocated through the per cpu

 * allocator or for addresses of per cpu variables.

 *

 * These operation guarantee exclusivity of access for other operations

 * on the *same* processor. The assumption is that per cpu data is only

 * accessed by a single processor instance (the current one).

 *

 * The arch code can provide optimized implementation by defining macros

 * for certain scalar sizes. F.e. provide this_cpu_add_2() to provide per

 * cpu atomic operations for 2 byte sized RMW actions. If arch code does

 * not provide operations for a scalar size then the fallback in the

 * generic code will be used.

 *

 * cmpxchg_double replaces two adjacent scalars at once.  The first two

 * parameters are per cpu variables which have to be of the same size.  A

 * truth value is returned to indicate success or failure (since a double

 * register result is difficult to handle).  There is very limited hardware

 * support for these operations, so only certain sizes may work.

 */

/*

 * Operations for contexts where we do not want to do any checks for

 * preemptions.  Unless strictly necessary, always use [__]this_cpu_*()

 * instead.

 *

 * If there is no other protection through preempt disable and/or disabling

 * interupts then one of these RMW operations can show unexpected behavior

 * because the execution thread was rescheduled on another processor or an

 * interrupt occurred and the same percpu variable was modified from the

 * interrupt context.

 */

#define raw_cpu_read(pcp)		__pcpu_size_call_return(raw_cpu_read_, pcp)

#define raw_cpu_write(pcp, val)		__pcpu_size_call(raw_cpu_write_, pcp, val)

#define raw_cpu_add(pcp, val)		__pcpu_size_call(raw_cpu_add_, pcp, val)

#define raw_cpu_and(pcp, val)		__pcpu_size_call(raw_cpu_and_, pcp, val)

#define raw_cpu_or(pcp, val)		__pcpu_size_call(raw_cpu_or_, pcp, val)

#define raw_cpu_add_return(pcp, val)	__pcpu_size_call_return2(raw_cpu_add_return_, pcp, val)

#define raw_cpu_xchg(pcp, nval)		__pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval)

#define raw_cpu_cmpxchg(pcp, oval, nval) \

	__pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval)

#define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \

	__pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)

#define raw_cpu_sub(pcp, val)		raw_cpu_add(pcp, -(val))

#define raw_cpu_inc(pcp)		raw_cpu_add(pcp, 1)

#define raw_cpu_dec(pcp)		raw_cpu_sub(pcp, 1)

#define raw_cpu_sub_return(pcp, val)	raw_cpu_add_return(pcp, -(typeof(pcp))(val))

#define raw_cpu_inc_return(pcp)		raw_cpu_add_return(pcp, 1)

#define raw_cpu_dec_return(pcp)		raw_cpu_add_return(pcp, -1)

/*

 * Operations for contexts that are safe from preemption/interrupts.  These

 * operations verify that preemption is disabled.

 */

#define __this_cpu_read(pcp)						\

({									\

	__this_cpu_preempt_check("read");				\

	raw_cpu_read(pcp);						\

})

#define __this_cpu_write(pcp, val)					\

({									\

	__this_cpu_preempt_check("write");				\

	raw_cpu_write(pcp, val);					\

})

#define __this_cpu_add(pcp, val)					\

({									\

	__this_cpu_preempt_check("add");				\

	raw_cpu_add(pcp, val);						\

})

#define __this_cpu_and(pcp, val)					\

({									\

	__this_cpu_preempt_check("and");				\

	raw_cpu_and(pcp, val);						\

})

#define __this_cpu_or(pcp, val)						\

({									\

	__this_cpu_preempt_check("or");					\

	raw_cpu_or(pcp, val);						\

})

#define __this_cpu_add_return(pcp, val)					\

({									\

	__this_cpu_preempt_check("add_return");				\

	raw_cpu_add_return(pcp, val);					\

})

#define __this_cpu_xchg(pcp, nval)					\

({									\

	__this_cpu_preempt_check("xchg");				\

	raw_cpu_xchg(pcp, nval);					\

})

#define __this_cpu_cmpxchg(pcp, oval, nval)				\

({									\

	__this_cpu_preempt_check("cmpxchg");				\

	raw_cpu_cmpxchg(pcp, oval, nval);				\

})

#define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \

({	__this_cpu_preempt_check("cmpxchg_double");			\

	raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2);	\

})

#define __this_cpu_sub(pcp, val)	__this_cpu_add(pcp, -(typeof(pcp))(val))

#define __this_cpu_inc(pcp)		__this_cpu_add(pcp, 1)

#define __this_cpu_dec(pcp)		__this_cpu_sub(pcp, 1)

#define __this_cpu_sub_return(pcp, val)	__this_cpu_add_return(pcp, -(typeof(pcp))(val))

#define __this_cpu_inc_return(pcp)	__this_cpu_add_return(pcp, 1)

#define __this_cpu_dec_return(pcp)	__this_cpu_add_return(pcp, -1)

/*

 * Operations with implied preemption/interrupt protection.  These

 * operations can be used without worrying about preemption or interrupt.

 */

#define this_cpu_read(pcp)		__pcpu_size_call_return(this_cpu_read_, pcp)

#define this_cpu_write(pcp, val)	__pcpu_size_call(this_cpu_write_, pcp, val)

#define this_cpu_add(pcp, val)		__pcpu_size_call(this_cpu_add_, pcp, val)

#define this_cpu_and(pcp, val)		__pcpu_size_call(this_cpu_and_, pcp, val)

#define this_cpu_or(pcp, val)		__pcpu_size_call(this_cpu_or_, pcp, val)

#define this_cpu_add_return(pcp, val)	__pcpu_size_call_return2(this_cpu_add_return_, pcp, val)

#define this_cpu_xchg(pcp, nval)	__pcpu_size_call_return2(this_cpu_xchg_, pcp, nval)

#define this_cpu_cmpxchg(pcp, oval, nval) \

	__pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval)

#define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \

	__pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2)

#define this_cpu_sub(pcp, val)		this_cpu_add(pcp, -(typeof(pcp))(val))

#define this_cpu_inc(pcp)		this_cpu_add(pcp, 1)

#define this_cpu_dec(pcp)		this_cpu_sub(pcp, 1)

#define this_cpu_sub_return(pcp, val)	this_cpu_add_return(pcp, -(typeof(pcp))(val))

#define this_cpu_inc_return(pcp)	this_cpu_add_return(pcp, 1)

#define this_cpu_dec_return(pcp)	this_cpu_add_return(pcp, -1)

#endif /* __ASSEMBLY__ */

#endif /* _LINUX_PERCPU_DEFS_H */