Commit 1d9bd516 authored by Tejun Heo's avatar Tejun Heo Committed by Jens Axboe

blk-mq: replace timeout synchronization with a RCU and generation based scheme

Currently, blk-mq timeout path synchronizes against the usual
issue/completion path using a complex scheme involving atomic
bitflags, REQ_ATOM_*, memory barriers and subtle memory coherence
rules.  Unfortunately, it contains quite a few holes.

There's a complex dancing around REQ_ATOM_STARTED and
REQ_ATOM_COMPLETE between issue/completion and timeout paths; however,
they don't have a synchronization point across request recycle
instances and it isn't clear what the barriers add.
blk_mq_check_expired() can easily read STARTED from N-2'th iteration,
deadline from N-1'th, blk_mark_rq_complete() against Nth instance.

In fact, it's pretty easy to make blk_mq_check_expired() terminate a
later instance of a request.  If we induce 5 sec delay before
time_after_eq() test in blk_mq_check_expired(), shorten the timeout to
2s, and issue back-to-back large IOs, blk-mq starts timing out
requests spuriously pretty quickly.  Nothing actually timed out.  It
just made the call on a recycle instance of a request and then
terminated a later instance long after the original instance finished.
The scenario isn't theoretical either.

This patch replaces the broken synchronization mechanism with a RCU
and generation number based one.

1. Each request has a u64 generation + state value, which can be
   updated only by the request owner.  Whenever a request becomes
   in-flight, the generation number gets bumped up too.  This provides
   the basis for the timeout path to distinguish different recycle
   instances of the request.

   Also, marking a request in-flight and setting its deadline are
   protected with a seqcount so that the timeout path can fetch both
   values coherently.

2. The timeout path fetches the generation, state and deadline.  If
   the verdict is timeout, it records the generation into a dedicated
   request abortion field and does RCU wait.

3. The completion path is also protected by RCU (from the previous
   patch) and checks whether the current generation number and state
   match the abortion field.  If so, it skips completion.

4. The timeout path, after RCU wait, scans requests again and
   terminates the ones whose generation and state still match the ones
   requested for abortion.

   By now, the timeout path knows that either the generation number
   and state changed if it lost the race or the completion will yield
   to it and can safely timeout the request.

While it's more lines of code, it's conceptually simpler, doesn't
depend on direct use of subtle memory ordering or coherence, and
hopefully doesn't terminate the wrong instance.

While this change makes REQ_ATOM_COMPLETE synchronization unnecessary
between issue/complete and timeout paths, REQ_ATOM_COMPLETE isn't
removed yet as it's still used in other places.  Future patches will
move all state tracking to the new mechanism and remove all bitops in
the hot paths.

Note that this patch adds a comment explaining a race condition in
BLK_EH_RESET_TIMER path.  The race has always been there and this
patch doesn't change it.  It's just documenting the existing race.

v2: - Fixed BLK_EH_RESET_TIMER handling as pointed out by Jianchao.
    - s/request->gstate_seqc/request->gstate_seq/ as suggested by Peter.
    - READ_ONCE() added in blk_mq_rq_update_state() as suggested by Peter.

v3: - Fixed possible extended seqcount / u64_stats_sync read looping
      spotted by Peter.
    - MQ_RQ_IDLE was incorrectly being set in complete_request instead
      of free_request.  Fixed.

v4: - Rebased on top of hctx_lock() refactoring patch.
    - Added comment explaining the use of hctx_lock() in completion path.

v5: - Added comments requested by Bart.
    - Note the addition of BLK_EH_RESET_TIMER race condition in the
      commit message.
Signed-off-by: default avatarTejun Heo <tj@kernel.org>
Cc: "jianchao.wang" <jianchao.w.wang@oracle.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Bart Van Assche <Bart.VanAssche@wdc.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5197c05e
...@@ -126,6 +126,8 @@ void blk_rq_init(struct request_queue *q, struct request *rq) ...@@ -126,6 +126,8 @@ void blk_rq_init(struct request_queue *q, struct request *rq)
rq->start_time = jiffies; rq->start_time = jiffies;
set_start_time_ns(rq); set_start_time_ns(rq);
rq->part = NULL; rq->part = NULL;
seqcount_init(&rq->gstate_seq);
u64_stats_init(&rq->aborted_gstate_sync);
} }
EXPORT_SYMBOL(blk_rq_init); EXPORT_SYMBOL(blk_rq_init);
......
...@@ -483,6 +483,7 @@ void blk_mq_free_request(struct request *rq) ...@@ -483,6 +483,7 @@ void blk_mq_free_request(struct request *rq)
if (blk_rq_rl(rq)) if (blk_rq_rl(rq))
blk_put_rl(blk_rq_rl(rq)); blk_put_rl(blk_rq_rl(rq));
blk_mq_rq_update_state(rq, MQ_RQ_IDLE);
clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags); clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags); clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
if (rq->tag != -1) if (rq->tag != -1)
...@@ -530,6 +531,8 @@ static void __blk_mq_complete_request(struct request *rq) ...@@ -530,6 +531,8 @@ static void __blk_mq_complete_request(struct request *rq)
bool shared = false; bool shared = false;
int cpu; int cpu;
WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT);
if (rq->internal_tag != -1) if (rq->internal_tag != -1)
blk_mq_sched_completed_request(rq); blk_mq_sched_completed_request(rq);
if (rq->rq_flags & RQF_STATS) { if (rq->rq_flags & RQF_STATS) {
...@@ -573,6 +576,36 @@ static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx) ...@@ -573,6 +576,36 @@ static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)
*srcu_idx = srcu_read_lock(hctx->queue_rq_srcu); *srcu_idx = srcu_read_lock(hctx->queue_rq_srcu);
} }
static void blk_mq_rq_update_aborted_gstate(struct request *rq, u64 gstate)
{
unsigned long flags;
/*
* blk_mq_rq_aborted_gstate() is used from the completion path and
* can thus be called from irq context. u64_stats_fetch in the
* middle of update on the same CPU leads to lockup. Disable irq
* while updating.
*/
local_irq_save(flags);
u64_stats_update_begin(&rq->aborted_gstate_sync);
rq->aborted_gstate = gstate;
u64_stats_update_end(&rq->aborted_gstate_sync);
local_irq_restore(flags);
}
static u64 blk_mq_rq_aborted_gstate(struct request *rq)
{
unsigned int start;
u64 aborted_gstate;
do {
start = u64_stats_fetch_begin(&rq->aborted_gstate_sync);
aborted_gstate = rq->aborted_gstate;
} while (u64_stats_fetch_retry(&rq->aborted_gstate_sync, start));
return aborted_gstate;
}
/** /**
* blk_mq_complete_request - end I/O on a request * blk_mq_complete_request - end I/O on a request
* @rq: the request being processed * @rq: the request being processed
...@@ -590,8 +623,20 @@ void blk_mq_complete_request(struct request *rq) ...@@ -590,8 +623,20 @@ void blk_mq_complete_request(struct request *rq)
if (unlikely(blk_should_fake_timeout(q))) if (unlikely(blk_should_fake_timeout(q)))
return; return;
/*
* If @rq->aborted_gstate equals the current instance, timeout is
* claiming @rq and we lost. This is synchronized through
* hctx_lock(). See blk_mq_timeout_work() for details.
*
* Completion path never blocks and we can directly use RCU here
* instead of hctx_lock() which can be either RCU or SRCU.
* However, that would complicate paths which want to synchronize
* against us. Let stay in sync with the issue path so that
* hctx_lock() covers both issue and completion paths.
*/
hctx_lock(hctx, &srcu_idx); hctx_lock(hctx, &srcu_idx);
if (!blk_mark_rq_complete(rq)) if (blk_mq_rq_aborted_gstate(rq) != rq->gstate &&
!blk_mark_rq_complete(rq))
__blk_mq_complete_request(rq); __blk_mq_complete_request(rq);
hctx_unlock(hctx, srcu_idx); hctx_unlock(hctx, srcu_idx);
} }
...@@ -617,34 +662,32 @@ void blk_mq_start_request(struct request *rq) ...@@ -617,34 +662,32 @@ void blk_mq_start_request(struct request *rq)
wbt_issue(q->rq_wb, &rq->issue_stat); wbt_issue(q->rq_wb, &rq->issue_stat);
} }
blk_add_timer(rq); WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
WARN_ON_ONCE(test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)); WARN_ON_ONCE(test_bit(REQ_ATOM_STARTED, &rq->atomic_flags));
/* /*
* Mark us as started and clear complete. Complete might have been * Mark @rq in-flight which also advances the generation number,
* set if requeue raced with timeout, which then marked it as * and register for timeout. Protect with a seqcount to allow the
* complete. So be sure to clear complete again when we start * timeout path to read both @rq->gstate and @rq->deadline
* the request, otherwise we'll ignore the completion event. * coherently.
* *
* Ensure that ->deadline is visible before we set STARTED, such that * This is the only place where a request is marked in-flight. If
* blk_mq_check_expired() is guaranteed to observe our ->deadline when * the timeout path reads an in-flight @rq->gstate, the
* it observes STARTED. * @rq->deadline it reads together under @rq->gstate_seq is
* guaranteed to be the matching one.
*/ */
smp_wmb(); preempt_disable();
write_seqcount_begin(&rq->gstate_seq);
blk_mq_rq_update_state(rq, MQ_RQ_IN_FLIGHT);
blk_add_timer(rq);
write_seqcount_end(&rq->gstate_seq);
preempt_enable();
set_bit(REQ_ATOM_STARTED, &rq->atomic_flags); set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) { if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
/*
* Coherence order guarantees these consecutive stores to a
* single variable propagate in the specified order. Thus the
* clear_bit() is ordered _after_ the set bit. See
* blk_mq_check_expired().
*
* (the bits must be part of the same byte for this to be
* true).
*/
clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags); clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
}
if (q->dma_drain_size && blk_rq_bytes(rq)) { if (q->dma_drain_size && blk_rq_bytes(rq)) {
/* /*
...@@ -677,6 +720,7 @@ static void __blk_mq_requeue_request(struct request *rq) ...@@ -677,6 +720,7 @@ static void __blk_mq_requeue_request(struct request *rq)
blk_mq_sched_requeue_request(rq); blk_mq_sched_requeue_request(rq);
if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
blk_mq_rq_update_state(rq, MQ_RQ_IDLE);
if (q->dma_drain_size && blk_rq_bytes(rq)) if (q->dma_drain_size && blk_rq_bytes(rq))
rq->nr_phys_segments--; rq->nr_phys_segments--;
} }
...@@ -774,6 +818,7 @@ EXPORT_SYMBOL(blk_mq_tag_to_rq); ...@@ -774,6 +818,7 @@ EXPORT_SYMBOL(blk_mq_tag_to_rq);
struct blk_mq_timeout_data { struct blk_mq_timeout_data {
unsigned long next; unsigned long next;
unsigned int next_set; unsigned int next_set;
unsigned int nr_expired;
}; };
void blk_mq_rq_timed_out(struct request *req, bool reserved) void blk_mq_rq_timed_out(struct request *req, bool reserved)
...@@ -801,6 +846,12 @@ void blk_mq_rq_timed_out(struct request *req, bool reserved) ...@@ -801,6 +846,12 @@ void blk_mq_rq_timed_out(struct request *req, bool reserved)
__blk_mq_complete_request(req); __blk_mq_complete_request(req);
break; break;
case BLK_EH_RESET_TIMER: case BLK_EH_RESET_TIMER:
/*
* As nothing prevents from completion happening while
* ->aborted_gstate is set, this may lead to ignored
* completions and further spurious timeouts.
*/
blk_mq_rq_update_aborted_gstate(req, 0);
blk_add_timer(req); blk_add_timer(req);
blk_clear_rq_complete(req); blk_clear_rq_complete(req);
break; break;
...@@ -816,50 +867,51 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, ...@@ -816,50 +867,51 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
struct request *rq, void *priv, bool reserved) struct request *rq, void *priv, bool reserved)
{ {
struct blk_mq_timeout_data *data = priv; struct blk_mq_timeout_data *data = priv;
unsigned long deadline; unsigned long gstate, deadline;
int start;
might_sleep();
if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
return; return;
/* /* read coherent snapshots of @rq->state_gen and @rq->deadline */
* Ensures that if we see STARTED we must also see our while (true) {
* up-to-date deadline, see blk_mq_start_request(). start = read_seqcount_begin(&rq->gstate_seq);
*/ gstate = READ_ONCE(rq->gstate);
smp_rmb(); deadline = rq->deadline;
if (!read_seqcount_retry(&rq->gstate_seq, start))
deadline = READ_ONCE(rq->deadline); break;
cond_resched();
/*
* The rq being checked may have been freed and reallocated
* out already here, we avoid this race by checking rq->deadline
* and REQ_ATOM_COMPLETE flag together:
*
* - if rq->deadline is observed as new value because of
* reusing, the rq won't be timed out because of timing.
* - if rq->deadline is observed as previous value,
* REQ_ATOM_COMPLETE flag won't be cleared in reuse path
* because we put a barrier between setting rq->deadline
* and clearing the flag in blk_mq_start_request(), so
* this rq won't be timed out too.
*/
if (time_after_eq(jiffies, deadline)) {
if (!blk_mark_rq_complete(rq)) {
/*
* Again coherence order ensures that consecutive reads
* from the same variable must be in that order. This
* ensures that if we see COMPLETE clear, we must then
* see STARTED set and we'll ignore this timeout.
*
* (There's also the MB implied by the test_and_clear())
*/
blk_mq_rq_timed_out(rq, reserved);
} }
/* if in-flight && overdue, mark for abortion */
if ((gstate & MQ_RQ_STATE_MASK) == MQ_RQ_IN_FLIGHT &&
time_after_eq(jiffies, deadline)) {
blk_mq_rq_update_aborted_gstate(rq, gstate);
data->nr_expired++;
hctx->nr_expired++;
} else if (!data->next_set || time_after(data->next, deadline)) { } else if (!data->next_set || time_after(data->next, deadline)) {
data->next = deadline; data->next = deadline;
data->next_set = 1; data->next_set = 1;
} }
} }
static void blk_mq_terminate_expired(struct blk_mq_hw_ctx *hctx,
struct request *rq, void *priv, bool reserved)
{
/*
* We marked @rq->aborted_gstate and waited for RCU. If there were
* completions that we lost to, they would have finished and
* updated @rq->gstate by now; otherwise, the completion path is
* now guaranteed to see @rq->aborted_gstate and yield. If
* @rq->aborted_gstate still matches @rq->gstate, @rq is ours.
*/
if (READ_ONCE(rq->gstate) == rq->aborted_gstate &&
!blk_mark_rq_complete(rq))
blk_mq_rq_timed_out(rq, reserved);
}
static void blk_mq_timeout_work(struct work_struct *work) static void blk_mq_timeout_work(struct work_struct *work)
{ {
struct request_queue *q = struct request_queue *q =
...@@ -867,7 +919,9 @@ static void blk_mq_timeout_work(struct work_struct *work) ...@@ -867,7 +919,9 @@ static void blk_mq_timeout_work(struct work_struct *work)
struct blk_mq_timeout_data data = { struct blk_mq_timeout_data data = {
.next = 0, .next = 0,
.next_set = 0, .next_set = 0,
.nr_expired = 0,
}; };
struct blk_mq_hw_ctx *hctx;
int i; int i;
/* A deadlock might occur if a request is stuck requiring a /* A deadlock might occur if a request is stuck requiring a
...@@ -886,14 +940,40 @@ static void blk_mq_timeout_work(struct work_struct *work) ...@@ -886,14 +940,40 @@ static void blk_mq_timeout_work(struct work_struct *work)
if (!percpu_ref_tryget(&q->q_usage_counter)) if (!percpu_ref_tryget(&q->q_usage_counter))
return; return;
/* scan for the expired ones and set their ->aborted_gstate */
blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data); blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
if (data.nr_expired) {
bool has_rcu = false;
/*
* Wait till everyone sees ->aborted_gstate. The
* sequential waits for SRCUs aren't ideal. If this ever
* becomes a problem, we can add per-hw_ctx rcu_head and
* wait in parallel.
*/
queue_for_each_hw_ctx(q, hctx, i) {
if (!hctx->nr_expired)
continue;
if (!(hctx->flags & BLK_MQ_F_BLOCKING))
has_rcu = true;
else
synchronize_srcu(hctx->queue_rq_srcu);
hctx->nr_expired = 0;
}
if (has_rcu)
synchronize_rcu();
/* terminate the ones we won */
blk_mq_queue_tag_busy_iter(q, blk_mq_terminate_expired, NULL);
}
if (data.next_set) { if (data.next_set) {
data.next = blk_rq_timeout(round_jiffies_up(data.next)); data.next = blk_rq_timeout(round_jiffies_up(data.next));
mod_timer(&q->timeout, data.next); mod_timer(&q->timeout, data.next);
} else { } else {
struct blk_mq_hw_ctx *hctx;
queue_for_each_hw_ctx(q, hctx, i) { queue_for_each_hw_ctx(q, hctx, i) {
/* the hctx may be unmapped, so check it here */ /* the hctx may be unmapped, so check it here */
if (blk_mq_hw_queue_mapped(hctx)) if (blk_mq_hw_queue_mapped(hctx))
...@@ -1893,6 +1973,22 @@ static size_t order_to_size(unsigned int order) ...@@ -1893,6 +1973,22 @@ static size_t order_to_size(unsigned int order)
return (size_t)PAGE_SIZE << order; return (size_t)PAGE_SIZE << order;
} }
static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
unsigned int hctx_idx, int node)
{
int ret;
if (set->ops->init_request) {
ret = set->ops->init_request(set, rq, hctx_idx, node);
if (ret)
return ret;
}
seqcount_init(&rq->gstate_seq);
u64_stats_init(&rq->aborted_gstate_sync);
return 0;
}
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
unsigned int hctx_idx, unsigned int depth) unsigned int hctx_idx, unsigned int depth)
{ {
...@@ -1954,13 +2050,10 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags, ...@@ -1954,13 +2050,10 @@ int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
struct request *rq = p; struct request *rq = p;
tags->static_rqs[i] = rq; tags->static_rqs[i] = rq;
if (set->ops->init_request) { if (blk_mq_init_request(set, rq, hctx_idx, node)) {
if (set->ops->init_request(set, rq, hctx_idx,
node)) {
tags->static_rqs[i] = NULL; tags->static_rqs[i] = NULL;
goto fail; goto fail;
} }
}
p += rq_size; p += rq_size;
i++; i++;
...@@ -2099,9 +2192,7 @@ static int blk_mq_init_hctx(struct request_queue *q, ...@@ -2099,9 +2192,7 @@ static int blk_mq_init_hctx(struct request_queue *q,
if (!hctx->fq) if (!hctx->fq)
goto sched_exit_hctx; goto sched_exit_hctx;
if (set->ops->init_request && if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx, node))
set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx,
node))
goto free_fq; goto free_fq;
if (hctx->flags & BLK_MQ_F_BLOCKING) if (hctx->flags & BLK_MQ_F_BLOCKING)
...@@ -3019,12 +3110,6 @@ static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie) ...@@ -3019,12 +3110,6 @@ static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
static int __init blk_mq_init(void) static int __init blk_mq_init(void)
{ {
/*
* See comment in block/blk.h rq_atomic_flags enum
*/
BUILD_BUG_ON((REQ_ATOM_STARTED / BITS_PER_BYTE) !=
(REQ_ATOM_COMPLETE / BITS_PER_BYTE));
cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL, cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
blk_mq_hctx_notify_dead); blk_mq_hctx_notify_dead);
return 0; return 0;
......
...@@ -27,6 +27,19 @@ struct blk_mq_ctx { ...@@ -27,6 +27,19 @@ struct blk_mq_ctx {
struct kobject kobj; struct kobject kobj;
} ____cacheline_aligned_in_smp; } ____cacheline_aligned_in_smp;
/*
* Bits for request->gstate. The lower two bits carry MQ_RQ_* state value
* and the upper bits the generation number.
*/
enum mq_rq_state {
MQ_RQ_IDLE = 0,
MQ_RQ_IN_FLIGHT = 1,
MQ_RQ_STATE_BITS = 2,
MQ_RQ_STATE_MASK = (1 << MQ_RQ_STATE_BITS) - 1,
MQ_RQ_GEN_INC = 1 << MQ_RQ_STATE_BITS,
};
void blk_mq_freeze_queue(struct request_queue *q); void blk_mq_freeze_queue(struct request_queue *q);
void blk_mq_free_queue(struct request_queue *q); void blk_mq_free_queue(struct request_queue *q);
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr); int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr);
...@@ -85,6 +98,39 @@ extern void blk_mq_rq_timed_out(struct request *req, bool reserved); ...@@ -85,6 +98,39 @@ extern void blk_mq_rq_timed_out(struct request *req, bool reserved);
void blk_mq_release(struct request_queue *q); void blk_mq_release(struct request_queue *q);
/**
* blk_mq_rq_state() - read the current MQ_RQ_* state of a request
* @rq: target request.
*/
static inline int blk_mq_rq_state(struct request *rq)
{
return READ_ONCE(rq->gstate) & MQ_RQ_STATE_MASK;
}
/**
* blk_mq_rq_update_state() - set the current MQ_RQ_* state of a request
* @rq: target request.
* @state: new state to set.
*
* Set @rq's state to @state. The caller is responsible for ensuring that
* there are no other updaters. A request can transition into IN_FLIGHT
* only from IDLE and doing so increments the generation number.
*/
static inline void blk_mq_rq_update_state(struct request *rq,
enum mq_rq_state state)
{
u64 old_val = READ_ONCE(rq->gstate);
u64 new_val = (old_val & ~MQ_RQ_STATE_MASK) | state;
if (state == MQ_RQ_IN_FLIGHT) {
WARN_ON_ONCE((old_val & MQ_RQ_STATE_MASK) != MQ_RQ_IDLE);
new_val += MQ_RQ_GEN_INC;
}
/* avoid exposing interim values */
WRITE_ONCE(rq->gstate, new_val);
}
static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q, static inline struct blk_mq_ctx *__blk_mq_get_ctx(struct request_queue *q,
unsigned int cpu) unsigned int cpu)
{ {
......
...@@ -208,7 +208,7 @@ void blk_add_timer(struct request *req) ...@@ -208,7 +208,7 @@ void blk_add_timer(struct request *req)
if (!req->timeout) if (!req->timeout)
req->timeout = q->rq_timeout; req->timeout = q->rq_timeout;
WRITE_ONCE(req->deadline, jiffies + req->timeout); req->deadline = jiffies + req->timeout;
/* /*
* Only the non-mq case needs to add the request to a protected list. * Only the non-mq case needs to add the request to a protected list.
......
...@@ -123,12 +123,6 @@ void blk_account_io_done(struct request *req); ...@@ -123,12 +123,6 @@ void blk_account_io_done(struct request *req);
* Internal atomic flags for request handling * Internal atomic flags for request handling
*/ */
enum rq_atomic_flags { enum rq_atomic_flags {
/*
* Keep these two bits first - not because we depend on the
* value of them, but we do depend on them being in the same
* byte of storage to ensure ordering on writes. Keeping them
* first will achieve that nicely.
*/
REQ_ATOM_COMPLETE = 0, REQ_ATOM_COMPLETE = 0,
REQ_ATOM_STARTED, REQ_ATOM_STARTED,
......
...@@ -51,6 +51,7 @@ struct blk_mq_hw_ctx { ...@@ -51,6 +51,7 @@ struct blk_mq_hw_ctx {
unsigned int queue_num; unsigned int queue_num;
atomic_t nr_active; atomic_t nr_active;
unsigned int nr_expired;
struct hlist_node cpuhp_dead; struct hlist_node cpuhp_dead;
struct kobject kobj; struct kobject kobj;
......
...@@ -27,6 +27,8 @@ ...@@ -27,6 +27,8 @@
#include <linux/percpu-refcount.h> #include <linux/percpu-refcount.h>
#include <linux/scatterlist.h> #include <linux/scatterlist.h>
#include <linux/blkzoned.h> #include <linux/blkzoned.h>
#include <linux/seqlock.h>
#include <linux/u64_stats_sync.h>
struct module; struct module;
struct scsi_ioctl_command; struct scsi_ioctl_command;
...@@ -230,6 +232,27 @@ struct request { ...@@ -230,6 +232,27 @@ struct request {
unsigned short write_hint; unsigned short write_hint;
/*
* On blk-mq, the lower bits of ->gstate (generation number and
* state) carry the MQ_RQ_* state value and the upper bits the
* generation number which is monotonically incremented and used to
* distinguish the reuse instances.
*
* ->gstate_seq allows updates to ->gstate and other fields
* (currently ->deadline) during request start to be read
* atomically from the timeout path, so that it can operate on a
* coherent set of information.
*/
seqcount_t gstate_seq;
u64 gstate;
/*
* ->aborted_gstate is used by the timeout to claim a specific
* recycle instance of this request. See blk_mq_timeout_work().
*/
struct u64_stats_sync aborted_gstate_sync;
u64 aborted_gstate;
unsigned long deadline; unsigned long deadline;
struct list_head timeout_list; struct list_head timeout_list;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment