Commit 8449eeda authored by Stefan Bühler's avatar Stefan Bühler Committed by Jens Axboe

io_uring: fix handling SQEs requesting NOWAIT

Not all request types set REQ_F_FORCE_NONBLOCK when they needed async
punting; reverse logic instead and set REQ_F_NOWAIT if request mustn't
be punted.
Signed-off-by: default avatarStefan Bühler <source@stbuehler.de>

Merged with my previous patch for this.
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 37624b58
...@@ -221,7 +221,7 @@ struct io_kiocb { ...@@ -221,7 +221,7 @@ struct io_kiocb {
struct list_head list; struct list_head list;
unsigned int flags; unsigned int flags;
refcount_t refs; refcount_t refs;
#define REQ_F_FORCE_NONBLOCK 1 /* inline submission attempt */ #define REQ_F_NOWAIT 1 /* must not punt to workers */
#define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */ #define REQ_F_IOPOLL_COMPLETED 2 /* polled IO has completed */
#define REQ_F_FIXED_FILE 4 /* ctx owns file */ #define REQ_F_FIXED_FILE 4 /* ctx owns file */
#define REQ_F_SEQ_PREV 8 /* sequential with previous */ #define REQ_F_SEQ_PREV 8 /* sequential with previous */
...@@ -774,10 +774,14 @@ static int io_prep_rw(struct io_kiocb *req, const struct sqe_submit *s, ...@@ -774,10 +774,14 @@ static int io_prep_rw(struct io_kiocb *req, const struct sqe_submit *s,
ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags)); ret = kiocb_set_rw_flags(kiocb, READ_ONCE(sqe->rw_flags));
if (unlikely(ret)) if (unlikely(ret))
return ret; return ret;
if (force_nonblock) {
/* don't allow async punt if RWF_NOWAIT was requested */
if (kiocb->ki_flags & IOCB_NOWAIT)
req->flags |= REQ_F_NOWAIT;
if (force_nonblock)
kiocb->ki_flags |= IOCB_NOWAIT; kiocb->ki_flags |= IOCB_NOWAIT;
req->flags |= REQ_F_FORCE_NONBLOCK;
}
if (ctx->flags & IORING_SETUP_IOPOLL) { if (ctx->flags & IORING_SETUP_IOPOLL) {
if (!(kiocb->ki_flags & IOCB_DIRECT) || if (!(kiocb->ki_flags & IOCB_DIRECT) ||
!kiocb->ki_filp->f_op->iopoll) !kiocb->ki_filp->f_op->iopoll)
...@@ -1436,8 +1440,7 @@ static void io_sq_wq_submit_work(struct work_struct *work) ...@@ -1436,8 +1440,7 @@ static void io_sq_wq_submit_work(struct work_struct *work)
struct sqe_submit *s = &req->submit; struct sqe_submit *s = &req->submit;
const struct io_uring_sqe *sqe = s->sqe; const struct io_uring_sqe *sqe = s->sqe;
/* Ensure we clear previously set forced non-block flag */ /* Ensure we clear previously set non-block flag */
req->flags &= ~REQ_F_FORCE_NONBLOCK;
req->rw.ki_flags &= ~IOCB_NOWAIT; req->rw.ki_flags &= ~IOCB_NOWAIT;
ret = 0; ret = 0;
...@@ -1623,7 +1626,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s, ...@@ -1623,7 +1626,7 @@ static int io_submit_sqe(struct io_ring_ctx *ctx, struct sqe_submit *s,
goto out; goto out;
ret = __io_submit_sqe(ctx, req, s, true); ret = __io_submit_sqe(ctx, req, s, true);
if (ret == -EAGAIN) { if (ret == -EAGAIN && !(req->flags & REQ_F_NOWAIT)) {
struct io_uring_sqe *sqe_copy; struct io_uring_sqe *sqe_copy;
sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL); sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment