364 lines
14 KiB
C
364 lines
14 KiB
C
/* SPDX-License-Identifier: GPL-2.0 */
|
|
#ifndef __ARCH_S390_PERCPU__
|
|
#define __ARCH_S390_PERCPU__
|
|
|
|
#include <linux/preempt.h>
|
|
#include <asm/cmpxchg.h>
|
|
#include <asm/march.h>
|
|
|
|
/*
|
|
* s390 uses its own implementation for per cpu data, the offset of
|
|
* the cpu local data area is cached in the cpu's lowcore memory.
|
|
*/
|
|
#define __my_cpu_offset get_lowcore()->percpu_offset
|
|
|
|
#define arch_raw_cpu_ptr(_ptr) \
|
|
({ \
|
|
unsigned long lc_percpu, tcp_ptr__; \
|
|
\
|
|
tcp_ptr__ = (__force unsigned long)(_ptr); \
|
|
lc_percpu = offsetof(struct lowcore, percpu_offset); \
|
|
asm_inline volatile( \
|
|
ALTERNATIVE("ag %[__ptr__],%[offzero](%%r0)\n", \
|
|
"ag %[__ptr__],%[offalt](%%r0)\n", \
|
|
ALT_FEATURE(MFEATURE_LOWCORE)) \
|
|
: [__ptr__] "+d" (tcp_ptr__) \
|
|
: [offzero] "i" (lc_percpu), \
|
|
[offalt] "i" (lc_percpu + LOWCORE_ALT_ADDRESS), \
|
|
"m" (((struct lowcore *)0)->percpu_offset) \
|
|
: "cc"); \
|
|
(TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \
|
|
})
|
|
|
|
/*
|
|
* We use a compare-and-swap loop since that uses less cpu cycles than
|
|
* disabling and enabling interrupts like the generic variant would do.
|
|
*/
|
|
#define arch_this_cpu_to_op_simple(pcp, val, op) \
|
|
({ \
|
|
typedef typeof(pcp) pcp_op_T__; \
|
|
pcp_op_T__ old__, new__, prev__; \
|
|
pcp_op_T__ *ptr__; \
|
|
preempt_disable_notrace(); \
|
|
ptr__ = raw_cpu_ptr(&(pcp)); \
|
|
prev__ = READ_ONCE(*ptr__); \
|
|
do { \
|
|
old__ = prev__; \
|
|
new__ = old__ op (val); \
|
|
prev__ = cmpxchg(ptr__, old__, new__); \
|
|
} while (prev__ != old__); \
|
|
preempt_enable_notrace(); \
|
|
new__; \
|
|
})
|
|
|
|
#define this_cpu_add_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
|
#define this_cpu_add_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
|
#define this_cpu_add_return_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
|
#define this_cpu_add_return_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
|
#define this_cpu_and_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
|
|
#define this_cpu_and_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
|
|
#define this_cpu_or_1(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
|
|
#define this_cpu_or_2(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
|
|
|
|
/*
|
|
* Macros to be used for percpu code section based on atomic instructions.
|
|
*
|
|
* Avoid the need to use preempt_disable() / preempt_disable() pairs and the
|
|
* conditional preempt_schedule_notrace() function calls which come with
|
|
* this. The idea is that this_cpu operations based on atomic instructions are
|
|
* guarded with mviy instructions:
|
|
*
|
|
* - The first mviy instruction writes the register number, which contains the
|
|
* percpu address variable to lowcore. This also indicates that a percpu
|
|
* code section is executed.
|
|
*
|
|
* - The first mviy instruction following the mviy instruction must be the ag
|
|
* instruction which adds the percpu offset to the percpu address register.
|
|
*
|
|
* - Afterwards the atomic percpu operation follows.
|
|
*
|
|
* - Then a second mviy instruction writes a zero to lowcore, which indicates
|
|
* the end of the percpu code section.
|
|
*
|
|
* - In case of an interrupt/exception/nmi the register number which was
|
|
* written to lowcore is copied to the exception frame (pt_regs), and a zero
|
|
* is written to lowcore.
|
|
*
|
|
* - On return to the previous context it is checked if a percpu code section
|
|
* was executed (saved register number not zero), and if the process was
|
|
* migrated to a different cpu. If the percpu offset was already added to
|
|
* the percpu address register (instruction address does _not_ point to the
|
|
* ag instruction) the content of the percpu address register is adjusted so
|
|
* it points to percpu variable of the new cpu.
|
|
*
|
|
* Inline assemblies making use of this typically have a code sequence like:
|
|
*
|
|
* MVIY_PERCPU(...) <- start of percpu code section
|
|
* AG_ALT(...) <- add percpu offset; must be the second instruction
|
|
* atomic_op <- atomic op
|
|
* MVIY_ALT(...) <- end of percpu code section
|
|
*/
|
|
|
|
#define MVIY_PERCPU(disp, dispalt, reg) \
|
|
".macro GEN_MVIY disp reg\n" \
|
|
".irp rs,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15\n" \
|
|
" .ifc \\reg,%%r\\rs\n" \
|
|
" mviy \\disp(%%r0),\\rs\n" \
|
|
" .endif\n" \
|
|
".endr\n" \
|
|
".endm\n" \
|
|
ALTERNATIVE("GEN_MVIY " __stringify(disp) " " __stringify(reg) "\n", \
|
|
"GEN_MVIY " __stringify(dispalt) " " __stringify(reg) "\n", \
|
|
ALT_FEATURE(MFEATURE_LOWCORE)) \
|
|
".purgem GEN_MVIY\n"
|
|
|
|
#define MVIY_ALT(disp, dispalt) \
|
|
ALTERNATIVE(" mviy " disp "(%%r0),0\n", \
|
|
" mviy " dispalt "(%%r0),0\n", \
|
|
ALT_FEATURE(MFEATURE_LOWCORE))
|
|
|
|
#define AG_ALT(disp, dispalt, reg) \
|
|
ALTERNATIVE(" ag " reg ", " disp "(%%r0)\n", \
|
|
" ag " reg ", " dispalt "(%%r0)\n", \
|
|
ALT_FEATURE(MFEATURE_LOWCORE))
|
|
|
|
#ifndef MARCH_HAS_Z196_FEATURES
|
|
|
|
#define this_cpu_add_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
|
#define this_cpu_add_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
|
#define this_cpu_add_return_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
|
#define this_cpu_add_return_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, +)
|
|
#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
|
|
#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, &)
|
|
#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
|
|
#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op_simple(pcp, val, |)
|
|
|
|
#else /* MARCH_HAS_Z196_FEATURES */
|
|
|
|
#define arch_this_cpu_add(pcp, val, op1, op2, szcast) \
|
|
do { \
|
|
unsigned long lc_pcpr, lc_pcpo; \
|
|
typedef typeof(pcp) pcp_op_T__; \
|
|
pcp_op_T__ val__ = (val); \
|
|
pcp_op_T__ old__, *ptr__; \
|
|
\
|
|
lc_pcpr = offsetof(struct lowcore, percpu_register); \
|
|
lc_pcpo = offsetof(struct lowcore, percpu_offset); \
|
|
ptr__ = PERCPU_PTR(&(pcp)); \
|
|
if (__builtin_constant_p(val__) && \
|
|
((szcast)val__ > -129) && ((szcast)val__ < 128)) { \
|
|
asm volatile( \
|
|
MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
|
|
AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \
|
|
op2 " 0(%[ptr__]),%[val__]\n" \
|
|
MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \
|
|
: [ptr__] "+&a" (ptr__), "+m" (*ptr__), \
|
|
"=m" (((struct lowcore *)0)->percpu_register) \
|
|
: [val__] "i" ((szcast)val__), \
|
|
[disppcpr] "i" (lc_pcpr), \
|
|
[disppcpo] "i" (lc_pcpo), \
|
|
[dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \
|
|
[dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \
|
|
"m" (((struct lowcore *)0)->percpu_offset) \
|
|
: "cc"); \
|
|
} else { \
|
|
asm volatile( \
|
|
MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
|
|
AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \
|
|
op1 " %[old__],%[val__],0(%[ptr__])\n" \
|
|
MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \
|
|
: [old__] "=&d" (old__), \
|
|
[ptr__] "+&a" (ptr__), "+m" (*ptr__), \
|
|
"=m" (((struct lowcore *)0)->percpu_register) \
|
|
: [val__] "d" (val__), \
|
|
[disppcpr] "i" (lc_pcpr), \
|
|
[disppcpo] "i" (lc_pcpo), \
|
|
[dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \
|
|
[dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \
|
|
"m" (((struct lowcore *)0)->percpu_offset) \
|
|
: "cc"); \
|
|
} \
|
|
} while (0)
|
|
|
|
#define this_cpu_add_4(pcp, val) arch_this_cpu_add(pcp, val, "laa", "asi", int)
|
|
#define this_cpu_add_8(pcp, val) arch_this_cpu_add(pcp, val, "laag", "agsi", long)
|
|
|
|
#define arch_this_cpu_add_return(pcp, val, op) \
|
|
({ \
|
|
unsigned long lc_pcpr, lc_pcpo; \
|
|
typedef typeof(pcp) pcp_op_T__; \
|
|
pcp_op_T__ val__ = (val); \
|
|
pcp_op_T__ old__, *ptr__; \
|
|
\
|
|
lc_pcpr = offsetof(struct lowcore, percpu_register); \
|
|
lc_pcpo = offsetof(struct lowcore, percpu_offset); \
|
|
ptr__ = PERCPU_PTR(&(pcp)); \
|
|
asm_inline volatile( \
|
|
MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
|
|
AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \
|
|
op " %[old__],%[val__],0(%[ptr__])\n" \
|
|
MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \
|
|
: [old__] "=&d" (old__), \
|
|
[ptr__] "+&a" (ptr__), "+m" (*ptr__), \
|
|
"=m" (((struct lowcore *)0)->percpu_register) \
|
|
: [val__] "d" (val__), \
|
|
[disppcpr] "i" (lc_pcpr), \
|
|
[disppcpo] "i" (lc_pcpo), \
|
|
[dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \
|
|
[dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \
|
|
"m" (((struct lowcore *)0)->percpu_offset) \
|
|
: "cc"); \
|
|
old__ + val__; \
|
|
})
|
|
|
|
#define this_cpu_add_return_4(pcp, val) arch_this_cpu_add_return(pcp, val, "laa")
|
|
#define this_cpu_add_return_8(pcp, val) arch_this_cpu_add_return(pcp, val, "laag")
|
|
|
|
#define arch_this_cpu_to_op(pcp, val, op) \
|
|
do { \
|
|
unsigned long lc_pcpr, lc_pcpo; \
|
|
typedef typeof(pcp) pcp_op_T__; \
|
|
pcp_op_T__ val__ = (val); \
|
|
pcp_op_T__ old__, *ptr__; \
|
|
\
|
|
lc_pcpr = offsetof(struct lowcore, percpu_register); \
|
|
lc_pcpo = offsetof(struct lowcore, percpu_offset); \
|
|
ptr__ = PERCPU_PTR(&(pcp)); \
|
|
asm_inline volatile( \
|
|
MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
|
|
AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \
|
|
op " %[old__],%[val__],0(%[ptr__])\n" \
|
|
MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \
|
|
: [old__] "=&d" (old__), \
|
|
[ptr__] "+&a" (ptr__), "+m" (*ptr__), \
|
|
"=m" (((struct lowcore *)0)->percpu_register) \
|
|
: [val__] "d" (val__), \
|
|
[disppcpr] "i" (lc_pcpr), \
|
|
[disppcpo] "i" (lc_pcpo), \
|
|
[dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \
|
|
[dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \
|
|
"m" (((struct lowcore *)0)->percpu_offset) \
|
|
: "cc"); \
|
|
} while (0)
|
|
|
|
#define this_cpu_and_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lan")
|
|
#define this_cpu_and_8(pcp, val) arch_this_cpu_to_op(pcp, val, "lang")
|
|
#define this_cpu_or_4(pcp, val) arch_this_cpu_to_op(pcp, val, "lao")
|
|
#define this_cpu_or_8(pcp, val) arch_this_cpu_to_op(pcp, val, "laog")
|
|
|
|
#endif /* MARCH_HAS_Z196_FEATURES */
|
|
|
|
#define arch_this_cpu_read(pcp, op) \
|
|
({ \
|
|
unsigned long lc_pcpr, lc_pcpo, res__; \
|
|
typedef typeof(pcp) pcp_op_T__; \
|
|
pcp_op_T__ *ptr__; \
|
|
\
|
|
lc_pcpr = offsetof(struct lowcore, percpu_register); \
|
|
lc_pcpo = offsetof(struct lowcore, percpu_offset); \
|
|
ptr__ = PERCPU_PTR(&(pcp)); \
|
|
asm_inline volatile( \
|
|
MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
|
|
AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \
|
|
op " %[res__],0(%[ptr__])\n" \
|
|
MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \
|
|
: [res__] "=&d" (res__), [ptr__] "+&a" (ptr__), \
|
|
"=m" (((struct lowcore *)0)->percpu_register) \
|
|
: [disppcpr] "i" (lc_pcpr), \
|
|
[disppcpo] "i" (lc_pcpo), \
|
|
[dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \
|
|
[dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \
|
|
"m" (*ptr__), \
|
|
"m" (((struct lowcore *)0)->percpu_offset) \
|
|
: "cc"); \
|
|
(pcp_op_T__)res__; \
|
|
})
|
|
|
|
#define this_cpu_read_1(pcp) arch_this_cpu_read(pcp, "llgc")
|
|
#define this_cpu_read_2(pcp) arch_this_cpu_read(pcp, "llgh")
|
|
#define this_cpu_read_4(pcp) arch_this_cpu_read(pcp, "llgf")
|
|
#define this_cpu_read_8(pcp) arch_this_cpu_read(pcp, "lg")
|
|
|
|
#define arch_this_cpu_write(pcp, val, op) \
|
|
do { \
|
|
unsigned long lc_pcpr, lc_pcpo; \
|
|
typedef typeof(pcp) pcp_op_T__; \
|
|
pcp_op_T__ *ptr__, val__ = (val); \
|
|
\
|
|
lc_pcpr = offsetof(struct lowcore, percpu_register); \
|
|
lc_pcpo = offsetof(struct lowcore, percpu_offset); \
|
|
ptr__ = PERCPU_PTR(&(pcp)); \
|
|
asm_inline volatile( \
|
|
MVIY_PERCPU("%[disppcpr]", "%[dispaltpcpr]", "%[ptr__]")\
|
|
AG_ALT("%[disppcpo]", "%[dispaltpcpo]", "%[ptr__]") \
|
|
op " %[val__],0(%[ptr__])\n" \
|
|
MVIY_ALT("%[disppcpr]", "%[dispaltpcpr]") \
|
|
: [ptr__] "+&a" (ptr__), "=m" (*ptr__), \
|
|
"=m" (((struct lowcore *)0)->percpu_register) \
|
|
: [val__] "d" (val__), \
|
|
[disppcpr] "i" (lc_pcpr), \
|
|
[disppcpo] "i" (lc_pcpo), \
|
|
[dispaltpcpr] "i" (lc_pcpr + LOWCORE_ALT_ADDRESS), \
|
|
[dispaltpcpo] "i" (lc_pcpo + LOWCORE_ALT_ADDRESS), \
|
|
"m" (((struct lowcore *)0)->percpu_offset) \
|
|
: "cc"); \
|
|
} while (0)
|
|
|
|
#define this_cpu_write_1(pcp, val) arch_this_cpu_write(pcp, val, "stc")
|
|
#define this_cpu_write_2(pcp, val) arch_this_cpu_write(pcp, val, "sth")
|
|
#define this_cpu_write_4(pcp, val) arch_this_cpu_write(pcp, val, "st")
|
|
#define this_cpu_write_8(pcp, val) arch_this_cpu_write(pcp, val, "stg")
|
|
|
|
#define arch_this_cpu_cmpxchg(pcp, oval, nval) \
|
|
({ \
|
|
typedef typeof(pcp) pcp_op_T__; \
|
|
pcp_op_T__ ret__; \
|
|
pcp_op_T__ *ptr__; \
|
|
preempt_disable_notrace(); \
|
|
ptr__ = raw_cpu_ptr(&(pcp)); \
|
|
ret__ = cmpxchg(ptr__, oval, nval); \
|
|
preempt_enable_notrace(); \
|
|
ret__; \
|
|
})
|
|
|
|
#define this_cpu_cmpxchg_1(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
|
|
#define this_cpu_cmpxchg_2(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
|
|
#define this_cpu_cmpxchg_4(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
|
|
#define this_cpu_cmpxchg_8(pcp, oval, nval) arch_this_cpu_cmpxchg(pcp, oval, nval)
|
|
|
|
#define this_cpu_cmpxchg64(pcp, o, n) this_cpu_cmpxchg_8(pcp, o, n)
|
|
|
|
#define this_cpu_cmpxchg128(pcp, oval, nval) \
|
|
({ \
|
|
typedef typeof(pcp) pcp_op_T__; \
|
|
u128 old__, new__, ret__; \
|
|
pcp_op_T__ *ptr__; \
|
|
old__ = oval; \
|
|
new__ = nval; \
|
|
preempt_disable_notrace(); \
|
|
ptr__ = raw_cpu_ptr(&(pcp)); \
|
|
ret__ = cmpxchg128((void *)ptr__, old__, new__); \
|
|
preempt_enable_notrace(); \
|
|
ret__; \
|
|
})
|
|
|
|
#define arch_this_cpu_xchg(pcp, nval) \
|
|
({ \
|
|
typeof(pcp) *ptr__; \
|
|
typeof(pcp) ret__; \
|
|
preempt_disable_notrace(); \
|
|
ptr__ = raw_cpu_ptr(&(pcp)); \
|
|
ret__ = xchg(ptr__, nval); \
|
|
preempt_enable_notrace(); \
|
|
ret__; \
|
|
})
|
|
|
|
#define this_cpu_xchg_1(pcp, nval) arch_this_cpu_xchg(pcp, nval)
|
|
#define this_cpu_xchg_2(pcp, nval) arch_this_cpu_xchg(pcp, nval)
|
|
#define this_cpu_xchg_4(pcp, nval) arch_this_cpu_xchg(pcp, nval)
|
|
#define this_cpu_xchg_8(pcp, nval) arch_this_cpu_xchg(pcp, nval)
|
|
|
|
#include <asm-generic/percpu.h>
|
|
|
|
#endif /* __ARCH_S390_PERCPU__ */
|