Skip to content

L
linux
Commit 4e15bb76 authored by Mathieu Desnoyers's avatar Mathieu Desnoyers Committed by Peter Zijlstra
Browse files
Options

selftests/rseq: x86-64: use %fs segment selector for accessing rseq thread area 

Rather than use rseq_get_abi() and pass its result through a register to
the inline assembler, directly access the per-thread rseq area through a
memory reference combining the %fs segment selector, the constant offset
of the field in struct rseq, and the rseq_offset value (in a register).
Signed-off-by: default avatarMathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: default avatarPeter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20220124171253.22072-15-mathieu.desnoyers@efficios.com
parent b53823fb
Hide whitespace changes
Inline Side-by-side
Showing with 30 additions and 28 deletions
tools/testing/selftests/rseq/rseq-x86.h
View file @ 4e15bb76
...@@ -28,6 +28,8 @@ ...@@ -28,6 +28,8 @@
#ifdef __x86_64__ #ifdef __x86_64__
#define RSEQ_ASM_TP_SEGMENT %%fs
#define rseq_smp_mb() \ #define rseq_smp_mb() \
__asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc") __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
#define rseq_smp_rmb() rseq_barrier() #define rseq_smp_rmb() rseq_barrier()
...@@ -123,14 +125,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) ...@@ -123,14 +125,14 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif #endif
/* Start rseq by storing table entry pointer into rseq_cs. */ /* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
RSEQ_INJECT_ASM(3) RSEQ_INJECT_ASM(3)
"cmpq %[v], %[expect]\n\t" "cmpq %[v], %[expect]\n\t"
"jnz %l[cmpfail]\n\t" "jnz %l[cmpfail]\n\t"
RSEQ_INJECT_ASM(4) RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE #ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
"cmpq %[v], %[expect]\n\t" "cmpq %[v], %[expect]\n\t"
"jnz %l[error2]\n\t" "jnz %l[error2]\n\t"
#endif #endif
...@@ -141,7 +143,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) ...@@ -141,7 +143,7 @@ int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
RSEQ_ASM_DEFINE_ABORT(4, "", abort) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
: /* gcc asm goto does not allow outputs */ : /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu), : [cpu_id] "r" (cpu),
[rseq_abi] "r" (rseq_get_abi()), [rseq_offset] "r" ((long)rseq_offset),
[v] "m" (*v), [v] "m" (*v),
[expect] "r" (expect), [expect] "r" (expect),
[newv] "r" (newv) [newv] "r" (newv)
...@@ -189,15 +191,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, ...@@ -189,15 +191,15 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif #endif
/* Start rseq by storing table entry pointer into rseq_cs. */ /* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
RSEQ_INJECT_ASM(3) RSEQ_INJECT_ASM(3)
"movq %[v], %%rbx\n\t" "movq %[v], %%rbx\n\t"
"cmpq %%rbx, %[expectnot]\n\t" "cmpq %%rbx, %[expectnot]\n\t"
"je %l[cmpfail]\n\t" "je %l[cmpfail]\n\t"
RSEQ_INJECT_ASM(4) RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE #ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
"movq %[v], %%rbx\n\t" "movq %[v], %%rbx\n\t"
"cmpq %%rbx, %[expectnot]\n\t" "cmpq %%rbx, %[expectnot]\n\t"
"je %l[error2]\n\t" "je %l[error2]\n\t"
...@@ -212,7 +214,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, ...@@ -212,7 +214,7 @@ int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
RSEQ_ASM_DEFINE_ABORT(4, "", abort) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
: /* gcc asm goto does not allow outputs */ : /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu), : [cpu_id] "r" (cpu),
[rseq_abi] "r" (rseq_get_abi()), [rseq_offset] "r" ((long)rseq_offset),
/* final store input */ /* final store input */
[v] "m" (*v), [v] "m" (*v),
[expectnot] "r" (expectnot), [expectnot] "r" (expectnot),
...@@ -255,11 +257,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) ...@@ -255,11 +257,11 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
#endif #endif
/* Start rseq by storing table entry pointer into rseq_cs. */ /* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
RSEQ_INJECT_ASM(3) RSEQ_INJECT_ASM(3)
#ifdef RSEQ_COMPARE_TWICE #ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
#endif #endif
/* final store */ /* final store */
"addq %[count], %[v]\n\t" "addq %[count], %[v]\n\t"
...@@ -268,7 +270,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu) ...@@ -268,7 +270,7 @@ int rseq_addv(intptr_t *v, intptr_t count, int cpu)
RSEQ_ASM_DEFINE_ABORT(4, "", abort) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
: /* gcc asm goto does not allow outputs */ : /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu), : [cpu_id] "r" (cpu),
[rseq_abi] "r" (rseq_get_abi()), [rseq_offset] "r" ((long)rseq_offset),
/* final store input */ /* final store input */
[v] "m" (*v), [v] "m" (*v),
[count] "er" (count) [count] "er" (count)
...@@ -309,11 +311,11 @@ int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu) ...@@ -309,11 +311,11 @@ int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
#endif #endif
/* Start rseq by storing table entry pointer into rseq_cs. */ /* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
RSEQ_INJECT_ASM(3) RSEQ_INJECT_ASM(3)
#ifdef RSEQ_COMPARE_TWICE #ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
#endif #endif
/* get p+v */ /* get p+v */
"movq %[ptr], %%rbx\n\t" "movq %[ptr], %%rbx\n\t"
...@@ -327,7 +329,7 @@ int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu) ...@@ -327,7 +329,7 @@ int rseq_offset_deref_addv(intptr_t *ptr, long off, intptr_t inc, int cpu)
RSEQ_ASM_DEFINE_ABORT(4, "", abort) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
: /* gcc asm goto does not allow outputs */ : /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu), : [cpu_id] "r" (cpu),
[rseq_abi] "r" (rseq_get_abi()), [rseq_offset] "r" ((long)rseq_offset),
/* final store input */ /* final store input */
[ptr] "m" (*ptr), [ptr] "m" (*ptr),
[off] "er" (off), [off] "er" (off),
...@@ -364,14 +366,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, ...@@ -364,14 +366,14 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
#endif #endif
/* Start rseq by storing table entry pointer into rseq_cs. */ /* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
RSEQ_INJECT_ASM(3) RSEQ_INJECT_ASM(3)
"cmpq %[v], %[expect]\n\t" "cmpq %[v], %[expect]\n\t"
"jnz %l[cmpfail]\n\t" "jnz %l[cmpfail]\n\t"
RSEQ_INJECT_ASM(4) RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE #ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
"cmpq %[v], %[expect]\n\t" "cmpq %[v], %[expect]\n\t"
"jnz %l[error2]\n\t" "jnz %l[error2]\n\t"
#endif #endif
...@@ -385,7 +387,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, ...@@ -385,7 +387,7 @@ int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
RSEQ_ASM_DEFINE_ABORT(4, "", abort) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
: /* gcc asm goto does not allow outputs */ : /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu), : [cpu_id] "r" (cpu),
[rseq_abi] "r" (rseq_get_abi()), [rseq_offset] "r" ((long)rseq_offset),
/* try store input */ /* try store input */
[v2] "m" (*v2), [v2] "m" (*v2),
[newv2] "r" (newv2), [newv2] "r" (newv2),
...@@ -444,8 +446,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, ...@@ -444,8 +446,8 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
#endif #endif
/* Start rseq by storing table entry pointer into rseq_cs. */ /* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
RSEQ_INJECT_ASM(3) RSEQ_INJECT_ASM(3)
"cmpq %[v], %[expect]\n\t" "cmpq %[v], %[expect]\n\t"
"jnz %l[cmpfail]\n\t" "jnz %l[cmpfail]\n\t"
...@@ -454,7 +456,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, ...@@ -454,7 +456,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
"jnz %l[cmpfail]\n\t" "jnz %l[cmpfail]\n\t"
RSEQ_INJECT_ASM(5) RSEQ_INJECT_ASM(5)
#ifdef RSEQ_COMPARE_TWICE #ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1]) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), %l[error1])
"cmpq %[v], %[expect]\n\t" "cmpq %[v], %[expect]\n\t"
"jnz %l[error2]\n\t" "jnz %l[error2]\n\t"
"cmpq %[v2], %[expect2]\n\t" "cmpq %[v2], %[expect2]\n\t"
...@@ -467,7 +469,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, ...@@ -467,7 +469,7 @@ int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
RSEQ_ASM_DEFINE_ABORT(4, "", abort) RSEQ_ASM_DEFINE_ABORT(4, "", abort)
: /* gcc asm goto does not allow outputs */ : /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu), : [cpu_id] "r" (cpu),
[rseq_abi] "r" (rseq_get_abi()), [rseq_offset] "r" ((long)rseq_offset),
/* cmp2 input */ /* cmp2 input */
[v2] "m" (*v2), [v2] "m" (*v2),
[expect2] "r" (expect2), [expect2] "r" (expect2),
...@@ -524,14 +526,14 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, ...@@ -524,14 +526,14 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
"movq %[dst], %[rseq_scratch1]\n\t" "movq %[dst], %[rseq_scratch1]\n\t"
"movq %[len], %[rseq_scratch2]\n\t" "movq %[len], %[rseq_scratch2]\n\t"
/* Start rseq by storing table entry pointer into rseq_cs. */ /* Start rseq by storing table entry pointer into rseq_cs. */
RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi])) RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_ASM_TP_SEGMENT:RSEQ_CS_OFFSET(%[rseq_offset]))
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 4f)
RSEQ_INJECT_ASM(3) RSEQ_INJECT_ASM(3)
"cmpq %[v], %[expect]\n\t" "cmpq %[v], %[expect]\n\t"
"jnz 5f\n\t" "jnz 5f\n\t"
RSEQ_INJECT_ASM(4) RSEQ_INJECT_ASM(4)
#ifdef RSEQ_COMPARE_TWICE #ifdef RSEQ_COMPARE_TWICE
RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f) RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_ASM_TP_SEGMENT:RSEQ_CPU_ID_OFFSET(%[rseq_offset]), 6f)
"cmpq %[v], %[expect]\n\t" "cmpq %[v], %[expect]\n\t"
"jnz 7f\n\t" "jnz 7f\n\t"
#endif #endif
...@@ -579,7 +581,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, ...@@ -579,7 +581,7 @@ int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
#endif #endif
: /* gcc asm goto does not allow outputs */ : /* gcc asm goto does not allow outputs */
: [cpu_id] "r" (cpu), : [cpu_id] "r" (cpu),
[rseq_abi] "r" (rseq_get_abi()), [rseq_offset] "r" ((long)rseq_offset),
/* final store input */ /* final store input */
[v] "m" (*v), [v] "m" (*v),
[expect] "r" (expect), [expect] "r" (expect),
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7