Skip to content

L
linux
Commit 51098f0e authored by Janis Schoetterl-Glausch's avatar Janis Schoetterl-Glausch Committed by Heiko Carstens
Browse files
Options

s390/cmpxchg: make loop condition for 1,2 byte cases precise 

The cmpxchg implementation for 1 and 2 bytes consists of a 4 byte
cmpxchg loop. Currently, the decision to retry is imprecise, looping if
bits outside the target byte(s) change instead of retrying until the
target byte(s) differ from the old value.
E.g. if an attempt to exchange (prev_left_0 old_bytes prev_right_0) is
made and it fails because the word at the address is
(prev_left_1 x prev_right_1) where both x != old_bytes and one of the
prev_*_1 values differs from the respective prev_*_0 value, the cmpxchg
is retried, even if by a semantic equivalent to a normal cmpxchg, the
exchange would fail.
Instead exit the loop if x != old_bytes and retry otherwise.
Signed-off-by: default avatarJanis Schoetterl-Glausch <scgl@linux.ibm.com>
Link: https://lore.kernel.org/r/20221116144711.3811011-1-scgl@linux.ibm.comSigned-off-by: default avatarHeiko Carstens <hca@linux.ibm.com>
parent 4148575a
Hide whitespace changes
Inline Side-by-side
Showing with 78 additions and 62 deletions
arch/s390/include/asm/cmpxchg.h
View file @ 51098f0e
...@@ -90,55 +90,63 @@ static __always_inline unsigned long __cmpxchg(unsigned long address, ...@@ -90,55 +90,63 @@ static __always_inline unsigned long __cmpxchg(unsigned long address,
{ {
switch (size) { switch (size) {
case 1: { case 1: {
unsigned int prev, tmp, shift; unsigned int prev, shift, mask;
shift = (3 ^ (address & 3)) << 3; shift = (3 ^ (address & 3)) << 3;
address ^= address & 3; address ^= address & 3;
old = (old & 0xff) << shift;
new = (new & 0xff) << shift;
mask = ~(0xff << shift);
asm volatile( asm volatile(
" l %[prev],%[address]\n" " l %[prev],%[address]\n"
"0: nr %[prev],%[mask]\n" " nr %[prev],%[mask]\n"
" lr %[tmp],%[prev]\n" " xilf %[mask],0xffffffff\n"
" or %[prev],%[old]\n" " or %[new],%[prev]\n"
" or %[tmp],%[new]\n" " or %[prev],%[tmp]\n"
" cs %[prev],%[tmp],%[address]\n" "0: lr %[tmp],%[prev]\n"
" cs %[prev],%[new],%[address]\n"
" jnl 1f\n" " jnl 1f\n"
" xr %[tmp],%[prev]\n" " xr %[tmp],%[prev]\n"
" xr %[new],%[tmp]\n"
" nr %[tmp],%[mask]\n" " nr %[tmp],%[mask]\n"
" jnz 0b\n" " jz 0b\n"
"1:" "1:"
: [prev] "=&d" (prev), : [prev] "=&d" (prev),
[tmp] "=&d" (tmp), [address] "+Q" (*(int *)address),
[address] "+Q" (*(int *)address) [tmp] "+&d" (old),
: [old] "d" ((old & 0xff) << shift), [new] "+&d" (new),
[new] "d" ((new & 0xff) << shift), [mask] "+&d" (mask)
[mask] "d" (~(0xff << shift)) :: "memory", "cc");
: "memory", "cc");
return prev >> shift; return prev >> shift;
} }
case 2: { case 2: {
unsigned int prev, tmp, shift; unsigned int prev, shift, mask;
shift = (2 ^ (address & 2)) << 3; shift = (2 ^ (address & 2)) << 3;
address ^= address & 2; address ^= address & 2;
old = (old & 0xffff) << shift;
new = (new & 0xffff) << shift;
mask = ~(0xffff << shift);
asm volatile( asm volatile(
" l %[prev],%[address]\n" " l %[prev],%[address]\n"
"0: nr %[prev],%[mask]\n" " nr %[prev],%[mask]\n"
" lr %[tmp],%[prev]\n" " xilf %[mask],0xffffffff\n"
" or %[prev],%[old]\n" " or %[new],%[prev]\n"
" or %[tmp],%[new]\n" " or %[prev],%[tmp]\n"
" cs %[prev],%[tmp],%[address]\n" "0: lr %[tmp],%[prev]\n"
" cs %[prev],%[new],%[address]\n"
" jnl 1f\n" " jnl 1f\n"
" xr %[tmp],%[prev]\n" " xr %[tmp],%[prev]\n"
" xr %[new],%[tmp]\n"
" nr %[tmp],%[mask]\n" " nr %[tmp],%[mask]\n"
" jnz 0b\n" " jz 0b\n"
"1:" "1:"
: [prev] "=&d" (prev), : [prev] "=&d" (prev),
[tmp] "=&d" (tmp), [address] "+Q" (*(int *)address),
[address] "+Q" (*(int *)address) [tmp] "+&d" (old),
: [old] "d" ((old & 0xffff) << shift), [new] "+&d" (new),
[new] "d" ((new & 0xffff) << shift), [mask] "+&d" (mask)
[mask] "d" (~(0xffff << shift)) :: "memory", "cc");
: "memory", "cc");
return prev >> shift; return prev >> shift;
} }
case 4: { case 4: {
......
arch/s390/include/asm/uaccess.h
View file @ 51098f0e
...@@ -400,74 +400,82 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, ...@@ -400,74 +400,82 @@ static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval,
switch (size) { switch (size) {
case 1: { case 1: {
unsigned int prev, tmp, shift; unsigned int prev, shift, mask, _old, _new;
shift = (3 ^ (address & 3)) << 3; shift = (3 ^ (address & 3)) << 3;
address ^= address & 3; address ^= address & 3;
_old = (old & 0xff) << shift;
_new = (new & 0xff) << shift;
mask = ~(0xff << shift);
asm volatile( asm volatile(
" spka 0(%[key])\n" " spka 0(%[key])\n"
" sacf 256\n" " sacf 256\n"
"0: l %[prev],%[address]\n" "0: l %[prev],%[address]\n"
"1: nr %[prev],%[mask]\n" "1: nr %[prev],%[mask]\n"
" lr %[tmp],%[prev]\n" " xilf %[mask],0xffffffff\n"
" or %[prev],%[old]\n" " or %[new],%[prev]\n"
" or %[tmp],%[new]\n" " or %[prev],%[tmp]\n"
"2: cs %[prev],%[tmp],%[address]\n" "2: lr %[tmp],%[prev]\n"
"3: jnl 4f\n" "3: cs %[prev],%[new],%[address]\n"
"4: jnl 5f\n"
" xr %[tmp],%[prev]\n" " xr %[tmp],%[prev]\n"
" xr %[new],%[tmp]\n"
" nr %[tmp],%[mask]\n" " nr %[tmp],%[mask]\n"
" jnz 1b\n" " jz 2b\n"
"4: sacf 768\n" "5: sacf 768\n"
" spka %[default_key]\n" " spka %[default_key]\n"
EX_TABLE_UA_LOAD_REG(0b, 4b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(1b, 4b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(2b, 4b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(3b, 4b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
: [rc] "+&d" (rc), : [rc] "+&d" (rc),
[prev] "=&d" (prev), [prev] "=&d" (prev),
[tmp] "=&d" (tmp), [address] "+Q" (*(int *)address),
[address] "+Q" (*(int *)address) [tmp] "+&d" (_old),
: [old] "d" (((unsigned int)old & 0xff) << shift), [new] "+&d" (_new),
[new] "d" (((unsigned int)new & 0xff) << shift), [mask] "+&d" (mask)
[mask] "d" (~(0xff << shift)), : [key] "a" (key << 4),
[key] "a" (key << 4),
[default_key] "J" (PAGE_DEFAULT_KEY) [default_key] "J" (PAGE_DEFAULT_KEY)
: "memory", "cc"); : "memory", "cc");
*(unsigned char *)uval = prev >> shift; *(unsigned char *)uval = prev >> shift;
return rc; return rc;
} }
case 2: { case 2: {
unsigned int prev, tmp, shift; unsigned int prev, shift, mask, _old, _new;
shift = (2 ^ (address & 2)) << 3; shift = (2 ^ (address & 2)) << 3;
address ^= address & 2; address ^= address & 2;
_old = (old & 0xffff) << shift;
_new = (new & 0xffff) << shift;
mask = ~(0xffff << shift);
asm volatile( asm volatile(
" spka 0(%[key])\n" " spka 0(%[key])\n"
" sacf 256\n" " sacf 256\n"
"0: l %[prev],%[address]\n" "0: l %[prev],%[address]\n"
"1: nr %[prev],%[mask]\n" "1: nr %[prev],%[mask]\n"
" lr %[tmp],%[prev]\n" " xilf %[mask],0xffffffff\n"
" or %[prev],%[old]\n" " or %[new],%[prev]\n"
" or %[tmp],%[new]\n" " or %[prev],%[tmp]\n"
"2: cs %[prev],%[tmp],%[address]\n" "2: lr %[tmp],%[prev]\n"
"3: jnl 4f\n" "3: cs %[prev],%[new],%[address]\n"
"4: jnl 5f\n"
" xr %[tmp],%[prev]\n" " xr %[tmp],%[prev]\n"
" xr %[new],%[tmp]\n"
" nr %[tmp],%[mask]\n" " nr %[tmp],%[mask]\n"
" jnz 1b\n" " jz 2b\n"
"4: sacf 768\n" "5: sacf 768\n"
" spka %[default_key]\n" " spka %[default_key]\n"
EX_TABLE_UA_LOAD_REG(0b, 4b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(1b, 4b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(2b, 4b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
EX_TABLE_UA_LOAD_REG(3b, 4b, %[rc], %[prev]) EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
: [rc] "+&d" (rc), : [rc] "+&d" (rc),
[prev] "=&d" (prev), [prev] "=&d" (prev),
[tmp] "=&d" (tmp), [address] "+Q" (*(int *)address),
[address] "+Q" (*(int *)address) [tmp] "+&d" (_old),
: [old] "d" (((unsigned int)old & 0xffff) << shift), [new] "+&d" (_new),
[new] "d" (((unsigned int)new & 0xffff) << shift), [mask] "+&d" (mask)
[mask] "d" (~(0xffff << shift)), : [key] "a" (key << 4),
[key] "a" (key << 4),
[default_key] "J" (PAGE_DEFAULT_KEY) [default_key] "J" (PAGE_DEFAULT_KEY)
: "memory", "cc"); : "memory", "cc");
*(unsigned short *)uval = prev >> shift; *(unsigned short *)uval = prev >> shift;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7