Commit c141b30e authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'rcu_urgent_for_5.8_rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip

Pull RCU-vs-KCSAN fixes from Borislav Petkov:
 "A single commit that uses "arch_" atomic operations to avoid the
  instrumentation that comes with the non-"arch_" versions.

  In preparation for that commit, it also has another commit that makes
  these "arch_" atomic operations available to generic code.

  Without these commits, KCSAN uses can see pointless errors"

* tag 'rcu_urgent_for_5.8_rc3' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip:
  rcu: Fixup noinstr warnings
  locking/atomics: Provide the arch_atomic_ interface to generic code
parents 7ecb59a5 b58e733f
This diff is collapsed.
...@@ -123,7 +123,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, ...@@ -123,7 +123,7 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val,
#ifdef CONFIG_DEBUG_ENTRY #ifdef CONFIG_DEBUG_ENTRY
/* Begin/end of an instrumentation safe region */ /* Begin/end of an instrumentation safe region */
#define instrumentation_begin() ({ \ #define instrumentation_begin() ({ \
asm volatile("%c0:\n\t" \ asm volatile("%c0: nop\n\t" \
".pushsection .discard.instr_begin\n\t" \ ".pushsection .discard.instr_begin\n\t" \
".long %c0b - .\n\t" \ ".long %c0b - .\n\t" \
".popsection\n\t" : : "i" (__COUNTER__)); \ ".popsection\n\t" : : "i" (__COUNTER__)); \
......
...@@ -250,7 +250,7 @@ static noinstr void rcu_dynticks_eqs_enter(void) ...@@ -250,7 +250,7 @@ static noinstr void rcu_dynticks_eqs_enter(void)
* next idle sojourn. * next idle sojourn.
*/ */
rcu_dynticks_task_trace_enter(); // Before ->dynticks update! rcu_dynticks_task_trace_enter(); // Before ->dynticks update!
seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
// RCU is no longer watching. Better be in extended quiescent state! // RCU is no longer watching. Better be in extended quiescent state!
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
(seq & RCU_DYNTICK_CTRL_CTR)); (seq & RCU_DYNTICK_CTRL_CTR));
...@@ -274,13 +274,13 @@ static noinstr void rcu_dynticks_eqs_exit(void) ...@@ -274,13 +274,13 @@ static noinstr void rcu_dynticks_eqs_exit(void)
* and we also must force ordering with the next RCU read-side * and we also must force ordering with the next RCU read-side
* critical section. * critical section.
*/ */
seq = atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks); seq = arch_atomic_add_return(RCU_DYNTICK_CTRL_CTR, &rdp->dynticks);
// RCU is now watching. Better not be in an extended quiescent state! // RCU is now watching. Better not be in an extended quiescent state!
rcu_dynticks_task_trace_exit(); // After ->dynticks update! rcu_dynticks_task_trace_exit(); // After ->dynticks update!
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) &&
!(seq & RCU_DYNTICK_CTRL_CTR)); !(seq & RCU_DYNTICK_CTRL_CTR));
if (seq & RCU_DYNTICK_CTRL_MASK) { if (seq & RCU_DYNTICK_CTRL_MASK) {
atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks); arch_atomic_andnot(RCU_DYNTICK_CTRL_MASK, &rdp->dynticks);
smp_mb__after_atomic(); /* _exit after clearing mask. */ smp_mb__after_atomic(); /* _exit after clearing mask. */
} }
} }
...@@ -313,7 +313,7 @@ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void) ...@@ -313,7 +313,7 @@ static __always_inline bool rcu_dynticks_curr_cpu_in_eqs(void)
{ {
struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
return !(atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR); return !(arch_atomic_read(&rdp->dynticks) & RCU_DYNTICK_CTRL_CTR);
} }
/* /*
...@@ -633,6 +633,10 @@ static noinstr void rcu_eqs_enter(bool user) ...@@ -633,6 +633,10 @@ static noinstr void rcu_eqs_enter(bool user)
do_nocb_deferred_wakeup(rdp); do_nocb_deferred_wakeup(rdp);
rcu_prepare_for_idle(); rcu_prepare_for_idle();
rcu_preempt_deferred_qs(current); rcu_preempt_deferred_qs(current);
// instrumentation for the noinstr rcu_dynticks_eqs_enter()
instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
instrumentation_end(); instrumentation_end();
WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */ WRITE_ONCE(rdp->dynticks_nesting, 0); /* Avoid irq-access tearing. */
// RCU is watching here ... // RCU is watching here ...
...@@ -692,6 +696,7 @@ noinstr void rcu_nmi_exit(void) ...@@ -692,6 +696,7 @@ noinstr void rcu_nmi_exit(void)
{ {
struct rcu_data *rdp = this_cpu_ptr(&rcu_data); struct rcu_data *rdp = this_cpu_ptr(&rcu_data);
instrumentation_begin();
/* /*
* Check for ->dynticks_nmi_nesting underflow and bad ->dynticks. * Check for ->dynticks_nmi_nesting underflow and bad ->dynticks.
* (We are exiting an NMI handler, so RCU better be paying attention * (We are exiting an NMI handler, so RCU better be paying attention
...@@ -705,7 +710,6 @@ noinstr void rcu_nmi_exit(void) ...@@ -705,7 +710,6 @@ noinstr void rcu_nmi_exit(void)
* leave it in non-RCU-idle state. * leave it in non-RCU-idle state.
*/ */
if (rdp->dynticks_nmi_nesting != 1) { if (rdp->dynticks_nmi_nesting != 1) {
instrumentation_begin();
trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2, trace_rcu_dyntick(TPS("--="), rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting - 2,
atomic_read(&rdp->dynticks)); atomic_read(&rdp->dynticks));
WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */ WRITE_ONCE(rdp->dynticks_nmi_nesting, /* No store tearing. */
...@@ -714,13 +718,15 @@ noinstr void rcu_nmi_exit(void) ...@@ -714,13 +718,15 @@ noinstr void rcu_nmi_exit(void)
return; return;
} }
instrumentation_begin();
/* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */ /* This NMI interrupted an RCU-idle CPU, restore RCU-idleness. */
trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks)); trace_rcu_dyntick(TPS("Startirq"), rdp->dynticks_nmi_nesting, 0, atomic_read(&rdp->dynticks));
WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */ WRITE_ONCE(rdp->dynticks_nmi_nesting, 0); /* Avoid store tearing. */
if (!in_nmi()) if (!in_nmi())
rcu_prepare_for_idle(); rcu_prepare_for_idle();
// instrumentation for the noinstr rcu_dynticks_eqs_enter()
instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
instrumentation_end(); instrumentation_end();
// RCU is watching here ... // RCU is watching here ...
...@@ -838,6 +844,10 @@ static void noinstr rcu_eqs_exit(bool user) ...@@ -838,6 +844,10 @@ static void noinstr rcu_eqs_exit(bool user)
rcu_dynticks_eqs_exit(); rcu_dynticks_eqs_exit();
// ... but is watching here. // ... but is watching here.
instrumentation_begin(); instrumentation_begin();
// instrumentation for the noinstr rcu_dynticks_eqs_exit()
instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
rcu_cleanup_after_idle(); rcu_cleanup_after_idle();
trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks)); trace_rcu_dyntick(TPS("End"), rdp->dynticks_nesting, 1, atomic_read(&rdp->dynticks));
WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current)); WARN_ON_ONCE(IS_ENABLED(CONFIG_RCU_EQS_DEBUG) && !user && !is_idle_task(current));
...@@ -983,13 +993,21 @@ noinstr void rcu_nmi_enter(void) ...@@ -983,13 +993,21 @@ noinstr void rcu_nmi_enter(void)
if (!in_nmi()) if (!in_nmi())
rcu_cleanup_after_idle(); rcu_cleanup_after_idle();
instrumentation_begin();
// instrumentation for the noinstr rcu_dynticks_curr_cpu_in_eqs()
instrument_atomic_read(&rdp->dynticks, sizeof(rdp->dynticks));
// instrumentation for the noinstr rcu_dynticks_eqs_exit()
instrument_atomic_write(&rdp->dynticks, sizeof(rdp->dynticks));
incby = 1; incby = 1;
} else if (!in_nmi()) { } else if (!in_nmi()) {
instrumentation_begin(); instrumentation_begin();
rcu_irq_enter_check_tick(); rcu_irq_enter_check_tick();
instrumentation_end(); instrumentation_end();
} else {
instrumentation_begin();
} }
instrumentation_begin();
trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="), trace_rcu_dyntick(incby == 1 ? TPS("Endirq") : TPS("++="),
rdp->dynticks_nmi_nesting, rdp->dynticks_nmi_nesting,
rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks)); rdp->dynticks_nmi_nesting + incby, atomic_read(&rdp->dynticks));
......
...@@ -58,6 +58,21 @@ cat << EOF ...@@ -58,6 +58,21 @@ cat << EOF
EOF EOF
} }
gen_proto_order_variant()
{
local meta="$1"; shift
local pfx="$1"; shift
local name="$1"; shift
local sfx="$1"; shift
local order="$1"; shift
local arch="$1"
local atomic="$2"
local basename="${arch}${atomic}_${pfx}${name}${sfx}"
printf "#define arch_${basename}${order} ${basename}${order}\n"
}
#gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...) #gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...)
gen_proto_order_variants() gen_proto_order_variants()
{ {
...@@ -72,6 +87,22 @@ gen_proto_order_variants() ...@@ -72,6 +87,22 @@ gen_proto_order_variants()
local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
if [ -z "$arch" ]; then
gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
if meta_has_acquire "${meta}"; then
gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
fi
if meta_has_release "${meta}"; then
gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
fi
if meta_has_relaxed "${meta}"; then
gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
fi
echo ""
fi
# If we don't have relaxed atomics, then we don't bother with ordering fallbacks # If we don't have relaxed atomics, then we don't bother with ordering fallbacks
# read_acquire and set_release need to be templated, though # read_acquire and set_release need to be templated, though
if ! meta_has_relaxed "${meta}"; then if ! meta_has_relaxed "${meta}"; then
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment