Commit 1a6b74fc authored by Jens Axboe's avatar Jens Axboe

io_uring: add general async offload context

Right now we just copy the sqe for async offload, but we want to store
more context across an async punt. In preparation for doing so, put the
sqe copy inside a structure that we can expand. With this pointer added,
we can get rid of REQ_F_FREE_SQE, as that is now indicated by whether
req->io is NULL or not.

No functional changes in this patch.
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 490547ca
...@@ -308,6 +308,10 @@ struct io_timeout { ...@@ -308,6 +308,10 @@ struct io_timeout {
struct io_timeout_data *data; struct io_timeout_data *data;
}; };
struct io_async_ctx {
struct io_uring_sqe sqe;
};
/* /*
* NOTE! Each of the iocb union members has the file pointer * NOTE! Each of the iocb union members has the file pointer
* as the first entry in their struct definition. So you can * as the first entry in their struct definition. So you can
...@@ -323,6 +327,7 @@ struct io_kiocb { ...@@ -323,6 +327,7 @@ struct io_kiocb {
}; };
const struct io_uring_sqe *sqe; const struct io_uring_sqe *sqe;
struct io_async_ctx *io;
struct file *ring_file; struct file *ring_file;
int ring_fd; int ring_fd;
bool has_user; bool has_user;
...@@ -353,7 +358,6 @@ struct io_kiocb { ...@@ -353,7 +358,6 @@ struct io_kiocb {
#define REQ_F_TIMEOUT_NOSEQ 8192 /* no timeout sequence */ #define REQ_F_TIMEOUT_NOSEQ 8192 /* no timeout sequence */
#define REQ_F_INFLIGHT 16384 /* on inflight list */ #define REQ_F_INFLIGHT 16384 /* on inflight list */
#define REQ_F_COMP_LOCKED 32768 /* completion under lock */ #define REQ_F_COMP_LOCKED 32768 /* completion under lock */
#define REQ_F_FREE_SQE 65536 /* free sqe if not async queued */
u64 user_data; u64 user_data;
u32 result; u32 result;
u32 sequence; u32 sequence;
...@@ -806,6 +810,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx, ...@@ -806,6 +810,7 @@ static struct io_kiocb *io_get_req(struct io_ring_ctx *ctx,
} }
got_it: got_it:
req->io = NULL;
req->ring_file = NULL; req->ring_file = NULL;
req->file = NULL; req->file = NULL;
req->ctx = ctx; req->ctx = ctx;
...@@ -836,8 +841,8 @@ static void __io_free_req(struct io_kiocb *req) ...@@ -836,8 +841,8 @@ static void __io_free_req(struct io_kiocb *req)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
if (req->flags & REQ_F_FREE_SQE) if (req->io)
kfree(req->sqe); kfree(req->io);
if (req->file && !(req->flags & REQ_F_FIXED_FILE)) if (req->file && !(req->flags & REQ_F_FIXED_FILE))
fput(req->file); fput(req->file);
if (req->flags & REQ_F_INFLIGHT) { if (req->flags & REQ_F_INFLIGHT) {
...@@ -1079,9 +1084,9 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events, ...@@ -1079,9 +1084,9 @@ static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
* completions for those, only batch free for fixed * completions for those, only batch free for fixed
* file and non-linked commands. * file and non-linked commands.
*/ */
if (((req->flags & if (((req->flags & (REQ_F_FIXED_FILE|REQ_F_LINK)) ==
(REQ_F_FIXED_FILE|REQ_F_LINK|REQ_F_FREE_SQE)) == REQ_F_FIXED_FILE) && !io_is_fallback_req(req) &&
REQ_F_FIXED_FILE) && !io_is_fallback_req(req)) { !req->io) {
reqs[to_free++] = req; reqs[to_free++] = req;
if (to_free == ARRAY_SIZE(reqs)) if (to_free == ARRAY_SIZE(reqs))
io_free_req_many(ctx, reqs, &to_free); io_free_req_many(ctx, reqs, &to_free);
...@@ -2259,7 +2264,7 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe, ...@@ -2259,7 +2264,7 @@ static int io_poll_add(struct io_kiocb *req, const struct io_uring_sqe *sqe,
if (!poll->wait) if (!poll->wait)
return -ENOMEM; return -ENOMEM;
req->sqe = NULL; req->io = NULL;
INIT_IO_WORK(&req->work, io_poll_complete_work); INIT_IO_WORK(&req->work, io_poll_complete_work);
events = READ_ONCE(sqe->poll_events); events = READ_ONCE(sqe->poll_events);
poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP; poll->events = demangle_poll(events) | EPOLLERR | EPOLLHUP;
...@@ -2602,27 +2607,27 @@ static int io_async_cancel(struct io_kiocb *req, const struct io_uring_sqe *sqe, ...@@ -2602,27 +2607,27 @@ static int io_async_cancel(struct io_kiocb *req, const struct io_uring_sqe *sqe,
static int io_req_defer(struct io_kiocb *req) static int io_req_defer(struct io_kiocb *req)
{ {
struct io_uring_sqe *sqe_copy;
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
struct io_async_ctx *io;
/* Still need defer if there is pending req in defer list. */ /* Still need defer if there is pending req in defer list. */
if (!req_need_defer(req) && list_empty(&ctx->defer_list)) if (!req_need_defer(req) && list_empty(&ctx->defer_list))
return 0; return 0;
sqe_copy = kmalloc(sizeof(*sqe_copy), GFP_KERNEL); io = kmalloc(sizeof(*io), GFP_KERNEL);
if (!sqe_copy) if (!io)
return -EAGAIN; return -EAGAIN;
spin_lock_irq(&ctx->completion_lock); spin_lock_irq(&ctx->completion_lock);
if (!req_need_defer(req) && list_empty(&ctx->defer_list)) { if (!req_need_defer(req) && list_empty(&ctx->defer_list)) {
spin_unlock_irq(&ctx->completion_lock); spin_unlock_irq(&ctx->completion_lock);
kfree(sqe_copy); kfree(io);
return 0; return 0;
} }
memcpy(sqe_copy, req->sqe, sizeof(*sqe_copy)); memcpy(&io->sqe, req->sqe, sizeof(io->sqe));
req->flags |= REQ_F_FREE_SQE; req->sqe = &io->sqe;
req->sqe = sqe_copy; req->io = io;
trace_io_uring_defer(ctx, req, req->user_data); trace_io_uring_defer(ctx, req, req->user_data);
list_add_tail(&req->list, &ctx->defer_list); list_add_tail(&req->list, &ctx->defer_list);
...@@ -2955,14 +2960,16 @@ static void __io_queue_sqe(struct io_kiocb *req) ...@@ -2955,14 +2960,16 @@ static void __io_queue_sqe(struct io_kiocb *req)
*/ */
if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) || if (ret == -EAGAIN && (!(req->flags & REQ_F_NOWAIT) ||
(req->flags & REQ_F_MUST_PUNT))) { (req->flags & REQ_F_MUST_PUNT))) {
struct io_uring_sqe *sqe_copy; struct io_async_ctx *io;
sqe_copy = kmemdup(req->sqe, sizeof(*sqe_copy), GFP_KERNEL); io = kmalloc(sizeof(*io), GFP_KERNEL);
if (!sqe_copy) if (!io)
goto err; goto err;
req->sqe = sqe_copy; memcpy(&io->sqe, req->sqe, sizeof(io->sqe));
req->flags |= REQ_F_FREE_SQE;
req->sqe = &io->sqe;
req->io = io;
if (req->work.flags & IO_WQ_WORK_NEEDS_FILES) { if (req->work.flags & IO_WQ_WORK_NEEDS_FILES) {
ret = io_grab_files(req); ret = io_grab_files(req);
...@@ -3063,7 +3070,7 @@ static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state, ...@@ -3063,7 +3070,7 @@ static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
*/ */
if (*link) { if (*link) {
struct io_kiocb *prev = *link; struct io_kiocb *prev = *link;
struct io_uring_sqe *sqe_copy; struct io_async_ctx *io;
if (req->sqe->flags & IOSQE_IO_DRAIN) if (req->sqe->flags & IOSQE_IO_DRAIN)
(*link)->flags |= REQ_F_DRAIN_LINK | REQ_F_IO_DRAIN; (*link)->flags |= REQ_F_DRAIN_LINK | REQ_F_IO_DRAIN;
...@@ -3079,14 +3086,15 @@ static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state, ...@@ -3079,14 +3086,15 @@ static void io_submit_sqe(struct io_kiocb *req, struct io_submit_state *state,
} }
} }
sqe_copy = kmemdup(req->sqe, sizeof(*sqe_copy), GFP_KERNEL); io = kmalloc(sizeof(*io), GFP_KERNEL);
if (!sqe_copy) { if (!io) {
ret = -EAGAIN; ret = -EAGAIN;
goto err_req; goto err_req;
} }
req->sqe = sqe_copy; memcpy(&io->sqe, req->sqe, sizeof(io->sqe));
req->flags |= REQ_F_FREE_SQE; req->sqe = &io->sqe;
req->io = io;
trace_io_uring_link(ctx, req, prev); trace_io_uring_link(ctx, req, prev);
list_add_tail(&req->list, &prev->link_list); list_add_tail(&req->list, &prev->link_list);
} else if (req->sqe->flags & IOSQE_IO_LINK) { } else if (req->sqe->flags & IOSQE_IO_LINK) {
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment