Commit 466c525d authored by Darrick J. Wong's avatar Darrick J. Wong

xfs: minimize overhead of drain wakeups by using jump labels

To reduce the runtime overhead even further when online fsck isn't
running, use a static branch key to decide if we call wake_up on the
drain.  For compilers that support jump labels, the call to wake_up is
replaced by a nop sled when nobody is waiting for intents to drain.

From my initial microbenchmarking, every transition of the static key
between the on and off states takes about 22000ns to complete; this is
paid entirely by the xfs_scrub process.  When the static key is off
(which it should be when fsck isn't running), the nop sled adds an
overhead of approximately 0.36ns to runtime code.  The post-atomic
lockless waiter check adds about 0.03ns, which is basically free.

For the few compilers that don't support jump labels, runtime code pays
the cost of calling wake_up on an empty waitqueue, which was observed to
be about 30ns.  However, most architectures that have sufficient memory
and CPU capacity to run XFS also support jump labels, so this is not
much of a worry.
Signed-off-by: default avatarDarrick J. Wong <djwong@kernel.org>
Reviewed-by: default avatarDave Chinner <dchinner@redhat.com>
parent 3f64c718
...@@ -95,6 +95,7 @@ config XFS_RT ...@@ -95,6 +95,7 @@ config XFS_RT
config XFS_DRAIN_INTENTS config XFS_DRAIN_INTENTS
bool bool
select JUMP_LABEL if HAVE_ARCH_JUMP_LABEL
config XFS_ONLINE_SCRUB config XFS_ONLINE_SCRUB
bool "XFS online metadata check support" bool "XFS online metadata check support"
......
...@@ -18,6 +18,15 @@ ...@@ -18,6 +18,15 @@
#include "scrub/scrub.h" #include "scrub/scrub.h"
#include "scrub/common.h" #include "scrub/common.h"
int
xchk_setup_agheader(
struct xfs_scrub *sc)
{
if (xchk_need_intent_drain(sc))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
return xchk_setup_fs(sc);
}
/* Superblock */ /* Superblock */
/* Cross-reference with the other btrees. */ /* Cross-reference with the other btrees. */
......
...@@ -24,6 +24,9 @@ int ...@@ -24,6 +24,9 @@ int
xchk_setup_ag_allocbt( xchk_setup_ag_allocbt(
struct xfs_scrub *sc) struct xfs_scrub *sc)
{ {
if (xchk_need_intent_drain(sc))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
return xchk_setup_ag_btree(sc, false); return xchk_setup_ag_btree(sc, false);
} }
......
...@@ -31,6 +31,9 @@ xchk_setup_inode_bmap( ...@@ -31,6 +31,9 @@ xchk_setup_inode_bmap(
{ {
int error; int error;
if (xchk_need_intent_drain(sc))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
error = xchk_get_inode(sc); error = xchk_get_inode(sc);
if (error) if (error)
goto out; goto out;
......
...@@ -487,6 +487,8 @@ xchk_perag_drain_and_lock( ...@@ -487,6 +487,8 @@ xchk_perag_drain_and_lock(
sa->agi_bp = NULL; sa->agi_bp = NULL;
} }
if (!(sc->flags & XCHK_FSGATES_DRAIN))
return -EDEADLOCK;
error = xfs_perag_intent_drain(sa->pag); error = xfs_perag_intent_drain(sa->pag);
if (error == -ERESTARTSYS) if (error == -ERESTARTSYS)
error = -EINTR; error = -EINTR;
...@@ -1005,3 +1007,25 @@ xchk_start_reaping( ...@@ -1005,3 +1007,25 @@ xchk_start_reaping(
} }
sc->flags &= ~XCHK_REAPING_DISABLED; sc->flags &= ~XCHK_REAPING_DISABLED;
} }
/*
* Enable filesystem hooks (i.e. runtime code patching) before starting a scrub
* operation. Callers must not hold any locks that intersect with the CPU
* hotplug lock (e.g. writeback locks) because code patching must halt the CPUs
* to change kernel code.
*/
void
xchk_fsgates_enable(
struct xfs_scrub *sc,
unsigned int scrub_fsgates)
{
ASSERT(!(scrub_fsgates & ~XCHK_FSGATES_ALL));
ASSERT(!(sc->flags & scrub_fsgates));
trace_xchk_fsgates_enable(sc, scrub_fsgates);
if (scrub_fsgates & XCHK_FSGATES_DRAIN)
xfs_drain_wait_enable();
sc->flags |= scrub_fsgates;
}
...@@ -72,6 +72,7 @@ bool xchk_should_check_xref(struct xfs_scrub *sc, int *error, ...@@ -72,6 +72,7 @@ bool xchk_should_check_xref(struct xfs_scrub *sc, int *error,
struct xfs_btree_cur **curpp); struct xfs_btree_cur **curpp);
/* Setup functions */ /* Setup functions */
int xchk_setup_agheader(struct xfs_scrub *sc);
int xchk_setup_fs(struct xfs_scrub *sc); int xchk_setup_fs(struct xfs_scrub *sc);
int xchk_setup_ag_allocbt(struct xfs_scrub *sc); int xchk_setup_ag_allocbt(struct xfs_scrub *sc);
int xchk_setup_ag_iallocbt(struct xfs_scrub *sc); int xchk_setup_ag_iallocbt(struct xfs_scrub *sc);
...@@ -151,4 +152,18 @@ int xchk_ilock_inverted(struct xfs_inode *ip, uint lock_mode); ...@@ -151,4 +152,18 @@ int xchk_ilock_inverted(struct xfs_inode *ip, uint lock_mode);
void xchk_stop_reaping(struct xfs_scrub *sc); void xchk_stop_reaping(struct xfs_scrub *sc);
void xchk_start_reaping(struct xfs_scrub *sc); void xchk_start_reaping(struct xfs_scrub *sc);
/*
* Setting up a hook to wait for intents to drain is costly -- we have to take
* the CPU hotplug lock and force an i-cache flush on all CPUs once to set it
* up, and again to tear it down. These costs add up quickly, so we only want
* to enable the drain waiter if the drain actually detected a conflict with
* running intent chains.
*/
static inline bool xchk_need_intent_drain(struct xfs_scrub *sc)
{
return sc->flags & XCHK_TRY_HARDER;
}
void xchk_fsgates_enable(struct xfs_scrub *sc, unsigned int scrub_fshooks);
#endif /* __XFS_SCRUB_COMMON_H__ */ #endif /* __XFS_SCRUB_COMMON_H__ */
...@@ -130,6 +130,13 @@ xchk_setup_fscounters( ...@@ -130,6 +130,13 @@ xchk_setup_fscounters(
struct xchk_fscounters *fsc; struct xchk_fscounters *fsc;
int error; int error;
/*
* If the AGF doesn't track btreeblks, we have to lock the AGF to count
* btree block usage by walking the actual btrees.
*/
if (!xfs_has_lazysbcount(sc->mp))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
sc->buf = kzalloc(sizeof(struct xchk_fscounters), XCHK_GFP_FLAGS); sc->buf = kzalloc(sizeof(struct xchk_fscounters), XCHK_GFP_FLAGS);
if (!sc->buf) if (!sc->buf)
return -ENOMEM; return -ENOMEM;
......
...@@ -32,6 +32,8 @@ int ...@@ -32,6 +32,8 @@ int
xchk_setup_ag_iallocbt( xchk_setup_ag_iallocbt(
struct xfs_scrub *sc) struct xfs_scrub *sc)
{ {
if (xchk_need_intent_drain(sc))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
return xchk_setup_ag_btree(sc, sc->flags & XCHK_TRY_HARDER); return xchk_setup_ag_btree(sc, sc->flags & XCHK_TRY_HARDER);
} }
......
...@@ -32,6 +32,9 @@ xchk_setup_inode( ...@@ -32,6 +32,9 @@ xchk_setup_inode(
{ {
int error; int error;
if (xchk_need_intent_drain(sc))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
/* /*
* Try to get the inode. If the verifiers fail, we try again * Try to get the inode. If the verifiers fail, we try again
* in raw mode. * in raw mode.
......
...@@ -53,6 +53,9 @@ xchk_setup_quota( ...@@ -53,6 +53,9 @@ xchk_setup_quota(
if (!xfs_this_quota_on(sc->mp, dqtype)) if (!xfs_this_quota_on(sc->mp, dqtype))
return -ENOENT; return -ENOENT;
if (xchk_need_intent_drain(sc))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
error = xchk_setup_fs(sc); error = xchk_setup_fs(sc);
if (error) if (error)
return error; return error;
......
...@@ -27,6 +27,8 @@ int ...@@ -27,6 +27,8 @@ int
xchk_setup_ag_refcountbt( xchk_setup_ag_refcountbt(
struct xfs_scrub *sc) struct xfs_scrub *sc)
{ {
if (xchk_need_intent_drain(sc))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
return xchk_setup_ag_btree(sc, false); return xchk_setup_ag_btree(sc, false);
} }
......
...@@ -24,6 +24,9 @@ int ...@@ -24,6 +24,9 @@ int
xchk_setup_ag_rmapbt( xchk_setup_ag_rmapbt(
struct xfs_scrub *sc) struct xfs_scrub *sc)
{ {
if (xchk_need_intent_drain(sc))
xchk_fsgates_enable(sc, XCHK_FSGATES_DRAIN);
return xchk_setup_ag_btree(sc, false); return xchk_setup_ag_btree(sc, false);
} }
......
...@@ -145,6 +145,21 @@ xchk_probe( ...@@ -145,6 +145,21 @@ xchk_probe(
/* Scrub setup and teardown */ /* Scrub setup and teardown */
static inline void
xchk_fsgates_disable(
struct xfs_scrub *sc)
{
if (!(sc->flags & XCHK_FSGATES_ALL))
return;
trace_xchk_fsgates_disable(sc, sc->flags & XCHK_FSGATES_ALL);
if (sc->flags & XCHK_FSGATES_DRAIN)
xfs_drain_wait_disable();
sc->flags &= ~XCHK_FSGATES_ALL;
}
/* Free all the resources and finish the transactions. */ /* Free all the resources and finish the transactions. */
STATIC int STATIC int
xchk_teardown( xchk_teardown(
...@@ -177,6 +192,8 @@ xchk_teardown( ...@@ -177,6 +192,8 @@ xchk_teardown(
kvfree(sc->buf); kvfree(sc->buf);
sc->buf = NULL; sc->buf = NULL;
} }
xchk_fsgates_disable(sc);
return error; return error;
} }
...@@ -191,25 +208,25 @@ static const struct xchk_meta_ops meta_scrub_ops[] = { ...@@ -191,25 +208,25 @@ static const struct xchk_meta_ops meta_scrub_ops[] = {
}, },
[XFS_SCRUB_TYPE_SB] = { /* superblock */ [XFS_SCRUB_TYPE_SB] = { /* superblock */
.type = ST_PERAG, .type = ST_PERAG,
.setup = xchk_setup_fs, .setup = xchk_setup_agheader,
.scrub = xchk_superblock, .scrub = xchk_superblock,
.repair = xrep_superblock, .repair = xrep_superblock,
}, },
[XFS_SCRUB_TYPE_AGF] = { /* agf */ [XFS_SCRUB_TYPE_AGF] = { /* agf */
.type = ST_PERAG, .type = ST_PERAG,
.setup = xchk_setup_fs, .setup = xchk_setup_agheader,
.scrub = xchk_agf, .scrub = xchk_agf,
.repair = xrep_agf, .repair = xrep_agf,
}, },
[XFS_SCRUB_TYPE_AGFL]= { /* agfl */ [XFS_SCRUB_TYPE_AGFL]= { /* agfl */
.type = ST_PERAG, .type = ST_PERAG,
.setup = xchk_setup_fs, .setup = xchk_setup_agheader,
.scrub = xchk_agfl, .scrub = xchk_agfl,
.repair = xrep_agfl, .repair = xrep_agfl,
}, },
[XFS_SCRUB_TYPE_AGI] = { /* agi */ [XFS_SCRUB_TYPE_AGI] = { /* agi */
.type = ST_PERAG, .type = ST_PERAG,
.setup = xchk_setup_fs, .setup = xchk_setup_agheader,
.scrub = xchk_agi, .scrub = xchk_agi,
.repair = xrep_agi, .repair = xrep_agi,
}, },
......
...@@ -96,9 +96,18 @@ struct xfs_scrub { ...@@ -96,9 +96,18 @@ struct xfs_scrub {
/* XCHK state flags grow up from zero, XREP state flags grown down from 2^31 */ /* XCHK state flags grow up from zero, XREP state flags grown down from 2^31 */
#define XCHK_TRY_HARDER (1 << 0) /* can't get resources, try again */ #define XCHK_TRY_HARDER (1 << 0) /* can't get resources, try again */
#define XCHK_REAPING_DISABLED (1 << 2) /* background block reaping paused */ #define XCHK_REAPING_DISABLED (1 << 1) /* background block reaping paused */
#define XCHK_FSGATES_DRAIN (1 << 2) /* defer ops draining enabled */
#define XREP_ALREADY_FIXED (1 << 31) /* checking our repair work */ #define XREP_ALREADY_FIXED (1 << 31) /* checking our repair work */
/*
* The XCHK_FSGATES* flags reflect functionality in the main filesystem that
* are only enabled for this particular online fsck. When not in use, the
* features are gated off via dynamic code patching, which is why the state
* must be enabled during scrub setup and can only be torn down afterwards.
*/
#define XCHK_FSGATES_ALL (XCHK_FSGATES_DRAIN)
/* Metadata scrubbers */ /* Metadata scrubbers */
int xchk_tester(struct xfs_scrub *sc); int xchk_tester(struct xfs_scrub *sc);
int xchk_superblock(struct xfs_scrub *sc); int xchk_superblock(struct xfs_scrub *sc);
......
...@@ -96,6 +96,12 @@ TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_FSCOUNTERS); ...@@ -96,6 +96,12 @@ TRACE_DEFINE_ENUM(XFS_SCRUB_TYPE_FSCOUNTERS);
{ XFS_SCRUB_OFLAG_WARNING, "warning" }, \ { XFS_SCRUB_OFLAG_WARNING, "warning" }, \
{ XFS_SCRUB_OFLAG_NO_REPAIR_NEEDED, "norepair" } { XFS_SCRUB_OFLAG_NO_REPAIR_NEEDED, "norepair" }
#define XFS_SCRUB_STATE_STRINGS \
{ XCHK_TRY_HARDER, "try_harder" }, \
{ XCHK_REAPING_DISABLED, "reaping_disabled" }, \
{ XCHK_FSGATES_DRAIN, "fsgates_drain" }, \
{ XREP_ALREADY_FIXED, "already_fixed" }
DECLARE_EVENT_CLASS(xchk_class, DECLARE_EVENT_CLASS(xchk_class,
TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_metadata *sm, TP_PROTO(struct xfs_inode *ip, struct xfs_scrub_metadata *sm,
int error), int error),
...@@ -142,6 +148,33 @@ DEFINE_SCRUB_EVENT(xchk_deadlock_retry); ...@@ -142,6 +148,33 @@ DEFINE_SCRUB_EVENT(xchk_deadlock_retry);
DEFINE_SCRUB_EVENT(xrep_attempt); DEFINE_SCRUB_EVENT(xrep_attempt);
DEFINE_SCRUB_EVENT(xrep_done); DEFINE_SCRUB_EVENT(xrep_done);
DECLARE_EVENT_CLASS(xchk_fsgate_class,
TP_PROTO(struct xfs_scrub *sc, unsigned int fsgate_flags),
TP_ARGS(sc, fsgate_flags),
TP_STRUCT__entry(
__field(dev_t, dev)
__field(unsigned int, type)
__field(unsigned int, fsgate_flags)
),
TP_fast_assign(
__entry->dev = sc->mp->m_super->s_dev;
__entry->type = sc->sm->sm_type;
__entry->fsgate_flags = fsgate_flags;
),
TP_printk("dev %d:%d type %s fsgates '%s'",
MAJOR(__entry->dev), MINOR(__entry->dev),
__print_symbolic(__entry->type, XFS_SCRUB_TYPE_STRINGS),
__print_flags(__entry->fsgate_flags, "|", XFS_SCRUB_STATE_STRINGS))
)
#define DEFINE_SCRUB_FSHOOK_EVENT(name) \
DEFINE_EVENT(xchk_fsgate_class, name, \
TP_PROTO(struct xfs_scrub *sc, unsigned int fsgates_flags), \
TP_ARGS(sc, fsgates_flags))
DEFINE_SCRUB_FSHOOK_EVENT(xchk_fsgates_enable);
DEFINE_SCRUB_FSHOOK_EVENT(xchk_fsgates_disable);
TRACE_EVENT(xchk_op_error, TRACE_EVENT(xchk_op_error,
TP_PROTO(struct xfs_scrub *sc, xfs_agnumber_t agno, TP_PROTO(struct xfs_scrub *sc, xfs_agnumber_t agno,
xfs_agblock_t bno, int error, void *ret_ip), xfs_agblock_t bno, int error, void *ret_ip),
......
...@@ -12,6 +12,31 @@ ...@@ -12,6 +12,31 @@
#include "xfs_ag.h" #include "xfs_ag.h"
#include "xfs_trace.h" #include "xfs_trace.h"
/*
* Use a static key here to reduce the overhead of xfs_drain_rele. If the
* compiler supports jump labels, the static branch will be replaced by a nop
* sled when there are no xfs_drain_wait callers. Online fsck is currently
* the only caller, so this is a reasonable tradeoff.
*
* Note: Patching the kernel code requires taking the cpu hotplug lock. Other
* parts of the kernel allocate memory with that lock held, which means that
* XFS callers cannot hold any locks that might be used by memory reclaim or
* writeback when calling the static_branch_{inc,dec} functions.
*/
static DEFINE_STATIC_KEY_FALSE(xfs_drain_waiter_gate);
void
xfs_drain_wait_disable(void)
{
static_branch_dec(&xfs_drain_waiter_gate);
}
void
xfs_drain_wait_enable(void)
{
static_branch_inc(&xfs_drain_waiter_gate);
}
void void
xfs_defer_drain_init( xfs_defer_drain_init(
struct xfs_defer_drain *dr) struct xfs_defer_drain *dr)
...@@ -46,6 +71,7 @@ static inline bool has_waiters(struct wait_queue_head *wq_head) ...@@ -46,6 +71,7 @@ static inline bool has_waiters(struct wait_queue_head *wq_head)
static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr) static inline void xfs_defer_drain_rele(struct xfs_defer_drain *dr)
{ {
if (atomic_dec_and_test(&dr->dr_count) && if (atomic_dec_and_test(&dr->dr_count) &&
static_branch_unlikely(&xfs_drain_waiter_gate) &&
has_waiters(&dr->dr_waiters)) has_waiters(&dr->dr_waiters))
wake_up(&dr->dr_waiters); wake_up(&dr->dr_waiters);
} }
......
...@@ -25,6 +25,9 @@ struct xfs_defer_drain { ...@@ -25,6 +25,9 @@ struct xfs_defer_drain {
void xfs_defer_drain_init(struct xfs_defer_drain *dr); void xfs_defer_drain_init(struct xfs_defer_drain *dr);
void xfs_defer_drain_free(struct xfs_defer_drain *dr); void xfs_defer_drain_free(struct xfs_defer_drain *dr);
void xfs_drain_wait_disable(void);
void xfs_drain_wait_enable(void);
/* /*
* Deferred Work Intent Drains * Deferred Work Intent Drains
* =========================== * ===========================
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment