Commit 23b3628e authored by Xiaoguang Wang's avatar Xiaoguang Wang Committed by Jens Axboe

io_uring: clear IORING_SQ_NEED_WAKEUP after executing task works

In io_sq_thread(), if there are task works to handle, current codes
will skip schedule() and go on polling sq again, but forget to clear
IORING_SQ_NEED_WAKEUP flag, fix this issue. Also add two helpers to
set and clear IORING_SQ_NEED_WAKEUP flag,
Signed-off-by: default avatarXiaoguang Wang <xiaoguang.wang@linux.alibaba.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 5af1d13e
...@@ -6344,6 +6344,21 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr, ...@@ -6344,6 +6344,21 @@ static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr,
return submitted; return submitted;
} }
static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
{
/* Tell userspace we may need a wakeup call */
spin_lock_irq(&ctx->completion_lock);
ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
spin_unlock_irq(&ctx->completion_lock);
}
static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
{
spin_lock_irq(&ctx->completion_lock);
ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
spin_unlock_irq(&ctx->completion_lock);
}
static int io_sq_thread(void *data) static int io_sq_thread(void *data)
{ {
struct io_ring_ctx *ctx = data; struct io_ring_ctx *ctx = data;
...@@ -6417,10 +6432,7 @@ static int io_sq_thread(void *data) ...@@ -6417,10 +6432,7 @@ static int io_sq_thread(void *data)
continue; continue;
} }
/* Tell userspace we may need a wakeup call */ io_ring_set_wakeup_flag(ctx);
spin_lock_irq(&ctx->completion_lock);
ctx->rings->sq_flags |= IORING_SQ_NEED_WAKEUP;
spin_unlock_irq(&ctx->completion_lock);
to_submit = io_sqring_entries(ctx); to_submit = io_sqring_entries(ctx);
if (!to_submit || ret == -EBUSY) { if (!to_submit || ret == -EBUSY) {
...@@ -6430,6 +6442,7 @@ static int io_sq_thread(void *data) ...@@ -6430,6 +6442,7 @@ static int io_sq_thread(void *data)
} }
if (io_run_task_work()) { if (io_run_task_work()) {
finish_wait(&ctx->sqo_wait, &wait); finish_wait(&ctx->sqo_wait, &wait);
io_ring_clear_wakeup_flag(ctx);
continue; continue;
} }
if (signal_pending(current)) if (signal_pending(current))
...@@ -6437,17 +6450,13 @@ static int io_sq_thread(void *data) ...@@ -6437,17 +6450,13 @@ static int io_sq_thread(void *data)
schedule(); schedule();
finish_wait(&ctx->sqo_wait, &wait); finish_wait(&ctx->sqo_wait, &wait);
spin_lock_irq(&ctx->completion_lock); io_ring_clear_wakeup_flag(ctx);
ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
spin_unlock_irq(&ctx->completion_lock);
ret = 0; ret = 0;
continue; continue;
} }
finish_wait(&ctx->sqo_wait, &wait); finish_wait(&ctx->sqo_wait, &wait);
spin_lock_irq(&ctx->completion_lock); io_ring_clear_wakeup_flag(ctx);
ctx->rings->sq_flags &= ~IORING_SQ_NEED_WAKEUP;
spin_unlock_irq(&ctx->completion_lock);
} }
mutex_lock(&ctx->uring_lock); mutex_lock(&ctx->uring_lock);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment