Commit 5730b27e authored by Jens Axboe's avatar Jens Axboe

io_uring: move cred assignment into io_issue_sqe()

If we move it in there, then we no longer have to care about it in io-wq.
This means we can drop the cred handling in io-wq, and we can drop the
REQ_F_WORK_INITIALIZED flag and async init functions as that was the last
user of it since we moved to the new workers. Then we can also drop
io_wq_work->creds, and just hold the personality u16 in there instead.
Suggested-by: default avatarPavel Begunkov <asml.silence@gmail.com>
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent 1575f21a
...@@ -52,9 +52,6 @@ struct io_worker { ...@@ -52,9 +52,6 @@ struct io_worker {
struct io_wq_work *cur_work; struct io_wq_work *cur_work;
spinlock_t lock; spinlock_t lock;
const struct cred *cur_creds;
const struct cred *saved_creds;
struct completion ref_done; struct completion ref_done;
struct completion started; struct completion started;
...@@ -180,11 +177,6 @@ static void io_worker_exit(struct io_worker *worker) ...@@ -180,11 +177,6 @@ static void io_worker_exit(struct io_worker *worker)
worker->flags = 0; worker->flags = 0;
preempt_enable(); preempt_enable();
if (worker->saved_creds) {
revert_creds(worker->saved_creds);
worker->cur_creds = worker->saved_creds = NULL;
}
raw_spin_lock_irq(&wqe->lock); raw_spin_lock_irq(&wqe->lock);
if (flags & IO_WORKER_F_FREE) if (flags & IO_WORKER_F_FREE)
hlist_nulls_del_rcu(&worker->nulls_node); hlist_nulls_del_rcu(&worker->nulls_node);
...@@ -326,10 +318,6 @@ static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker) ...@@ -326,10 +318,6 @@ static void __io_worker_idle(struct io_wqe *wqe, struct io_worker *worker)
worker->flags |= IO_WORKER_F_FREE; worker->flags |= IO_WORKER_F_FREE;
hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list); hlist_nulls_add_head_rcu(&worker->nulls_node, &wqe->free_list);
} }
if (worker->saved_creds) {
revert_creds(worker->saved_creds);
worker->cur_creds = worker->saved_creds = NULL;
}
} }
static inline unsigned int io_get_work_hash(struct io_wq_work *work) static inline unsigned int io_get_work_hash(struct io_wq_work *work)
...@@ -404,18 +392,6 @@ static void io_flush_signals(void) ...@@ -404,18 +392,6 @@ static void io_flush_signals(void)
} }
} }
static void io_wq_switch_creds(struct io_worker *worker,
struct io_wq_work *work)
{
const struct cred *old_creds = override_creds(work->creds);
worker->cur_creds = work->creds;
if (worker->saved_creds)
put_cred(old_creds); /* creds set by previous switch */
else
worker->saved_creds = old_creds;
}
static void io_assign_current_work(struct io_worker *worker, static void io_assign_current_work(struct io_worker *worker,
struct io_wq_work *work) struct io_wq_work *work)
{ {
...@@ -465,8 +441,6 @@ static void io_worker_handle_work(struct io_worker *worker) ...@@ -465,8 +441,6 @@ static void io_worker_handle_work(struct io_worker *worker)
unsigned int hash = io_get_work_hash(work); unsigned int hash = io_get_work_hash(work);
next_hashed = wq_next_work(work); next_hashed = wq_next_work(work);
if (work->creds && worker->cur_creds != work->creds)
io_wq_switch_creds(worker, work);
wq->do_work(work); wq->do_work(work);
io_assign_current_work(worker, NULL); io_assign_current_work(worker, NULL);
......
...@@ -79,8 +79,8 @@ static inline void wq_list_del(struct io_wq_work_list *list, ...@@ -79,8 +79,8 @@ static inline void wq_list_del(struct io_wq_work_list *list,
struct io_wq_work { struct io_wq_work {
struct io_wq_work_node list; struct io_wq_work_node list;
const struct cred *creds;
unsigned flags; unsigned flags;
unsigned short personality;
}; };
static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) static inline struct io_wq_work *wq_next_work(struct io_wq_work *work)
......
...@@ -688,7 +688,6 @@ enum { ...@@ -688,7 +688,6 @@ enum {
REQ_F_POLLED_BIT, REQ_F_POLLED_BIT,
REQ_F_BUFFER_SELECTED_BIT, REQ_F_BUFFER_SELECTED_BIT,
REQ_F_NO_FILE_TABLE_BIT, REQ_F_NO_FILE_TABLE_BIT,
REQ_F_WORK_INITIALIZED_BIT,
REQ_F_LTIMEOUT_ACTIVE_BIT, REQ_F_LTIMEOUT_ACTIVE_BIT,
REQ_F_COMPLETE_INLINE_BIT, REQ_F_COMPLETE_INLINE_BIT,
...@@ -730,8 +729,6 @@ enum { ...@@ -730,8 +729,6 @@ enum {
REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT),
/* doesn't need file table for this request */ /* doesn't need file table for this request */
REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT), REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT),
/* io_wq_work is initialized */
REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT),
/* linked timeout is active, i.e. prepared by link's head */ /* linked timeout is active, i.e. prepared by link's head */
REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT), REQ_F_LTIMEOUT_ACTIVE = BIT(REQ_F_LTIMEOUT_ACTIVE_BIT),
/* completion is deferred through io_comp_state */ /* completion is deferred through io_comp_state */
...@@ -1094,24 +1091,6 @@ static inline void req_set_fail_links(struct io_kiocb *req) ...@@ -1094,24 +1091,6 @@ static inline void req_set_fail_links(struct io_kiocb *req)
req->flags |= REQ_F_FAIL_LINK; req->flags |= REQ_F_FAIL_LINK;
} }
static inline void __io_req_init_async(struct io_kiocb *req)
{
memset(&req->work, 0, sizeof(req->work));
req->flags |= REQ_F_WORK_INITIALIZED;
}
/*
* Note: must call io_req_init_async() for the first time you
* touch any members of io_wq_work.
*/
static inline void io_req_init_async(struct io_kiocb *req)
{
if (req->flags & REQ_F_WORK_INITIALIZED)
return;
__io_req_init_async(req);
}
static void io_ring_ctx_ref_free(struct percpu_ref *ref) static void io_ring_ctx_ref_free(struct percpu_ref *ref)
{ {
struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs); struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
...@@ -1196,13 +1175,6 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq) ...@@ -1196,13 +1175,6 @@ static bool req_need_defer(struct io_kiocb *req, u32 seq)
static void io_req_clean_work(struct io_kiocb *req) static void io_req_clean_work(struct io_kiocb *req)
{ {
if (!(req->flags & REQ_F_WORK_INITIALIZED))
return;
if (req->work.creds) {
put_cred(req->work.creds);
req->work.creds = NULL;
}
if (req->flags & REQ_F_INFLIGHT) { if (req->flags & REQ_F_INFLIGHT) {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
struct io_uring_task *tctx = req->task->io_uring; struct io_uring_task *tctx = req->task->io_uring;
...@@ -1215,8 +1187,6 @@ static void io_req_clean_work(struct io_kiocb *req) ...@@ -1215,8 +1187,6 @@ static void io_req_clean_work(struct io_kiocb *req)
if (atomic_read(&tctx->in_idle)) if (atomic_read(&tctx->in_idle))
wake_up(&tctx->wait); wake_up(&tctx->wait);
} }
req->flags &= ~REQ_F_WORK_INITIALIZED;
} }
static void io_req_track_inflight(struct io_kiocb *req) static void io_req_track_inflight(struct io_kiocb *req)
...@@ -1224,7 +1194,6 @@ static void io_req_track_inflight(struct io_kiocb *req) ...@@ -1224,7 +1194,6 @@ static void io_req_track_inflight(struct io_kiocb *req)
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
if (!(req->flags & REQ_F_INFLIGHT)) { if (!(req->flags & REQ_F_INFLIGHT)) {
io_req_init_async(req);
req->flags |= REQ_F_INFLIGHT; req->flags |= REQ_F_INFLIGHT;
spin_lock_irq(&ctx->inflight_lock); spin_lock_irq(&ctx->inflight_lock);
...@@ -1238,8 +1207,6 @@ static void io_prep_async_work(struct io_kiocb *req) ...@@ -1238,8 +1207,6 @@ static void io_prep_async_work(struct io_kiocb *req)
const struct io_op_def *def = &io_op_defs[req->opcode]; const struct io_op_def *def = &io_op_defs[req->opcode];
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
io_req_init_async(req);
if (req->flags & REQ_F_FORCE_ASYNC) if (req->flags & REQ_F_FORCE_ASYNC)
req->work.flags |= IO_WQ_WORK_CONCURRENT; req->work.flags |= IO_WQ_WORK_CONCURRENT;
...@@ -1250,8 +1217,6 @@ static void io_prep_async_work(struct io_kiocb *req) ...@@ -1250,8 +1217,6 @@ static void io_prep_async_work(struct io_kiocb *req)
if (def->unbound_nonreg_file) if (def->unbound_nonreg_file)
req->work.flags |= IO_WQ_WORK_UNBOUND; req->work.flags |= IO_WQ_WORK_UNBOUND;
} }
if (!req->work.creds)
req->work.creds = get_current_cred();
} }
static void io_prep_async_link(struct io_kiocb *req) static void io_prep_async_link(struct io_kiocb *req)
...@@ -3578,7 +3543,6 @@ static int __io_splice_prep(struct io_kiocb *req, ...@@ -3578,7 +3543,6 @@ static int __io_splice_prep(struct io_kiocb *req,
* Splice operation will be punted aync, and here need to * Splice operation will be punted aync, and here need to
* modify io_wq_work.flags, so initialize io_wq_work firstly. * modify io_wq_work.flags, so initialize io_wq_work firstly.
*/ */
io_req_init_async(req);
req->work.flags |= IO_WQ_WORK_UNBOUND; req->work.flags |= IO_WQ_WORK_UNBOUND;
} }
...@@ -5935,8 +5899,22 @@ static void __io_clean_op(struct io_kiocb *req) ...@@ -5935,8 +5899,22 @@ static void __io_clean_op(struct io_kiocb *req)
static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
{ {
struct io_ring_ctx *ctx = req->ctx; struct io_ring_ctx *ctx = req->ctx;
const struct cred *creds = NULL;
int ret; int ret;
if (req->work.personality) {
const struct cred *new_creds;
if (!(issue_flags & IO_URING_F_NONBLOCK))
mutex_lock(&ctx->uring_lock);
new_creds = idr_find(&ctx->personality_idr, req->work.personality);
if (!(issue_flags & IO_URING_F_NONBLOCK))
mutex_unlock(&ctx->uring_lock);
if (!new_creds)
return -EINVAL;
creds = override_creds(new_creds);
}
switch (req->opcode) { switch (req->opcode) {
case IORING_OP_NOP: case IORING_OP_NOP:
ret = io_nop(req, issue_flags); ret = io_nop(req, issue_flags);
...@@ -6043,6 +6021,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) ...@@ -6043,6 +6021,9 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
break; break;
} }
if (creds)
revert_creds(creds);
if (ret) if (ret)
return ret; return ret;
...@@ -6206,18 +6187,10 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req) ...@@ -6206,18 +6187,10 @@ static struct io_kiocb *io_prep_linked_timeout(struct io_kiocb *req)
static void __io_queue_sqe(struct io_kiocb *req) static void __io_queue_sqe(struct io_kiocb *req)
{ {
struct io_kiocb *linked_timeout = io_prep_linked_timeout(req); struct io_kiocb *linked_timeout = io_prep_linked_timeout(req);
const struct cred *old_creds = NULL;
int ret; int ret;
if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds &&
req->work.creds != current_cred())
old_creds = override_creds(req->work.creds);
ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER); ret = io_issue_sqe(req, IO_URING_F_NONBLOCK|IO_URING_F_COMPLETE_DEFER);
if (old_creds)
revert_creds(old_creds);
/* /*
* We async punt it if the file wasn't marked NOWAIT, or if the file * We async punt it if the file wasn't marked NOWAIT, or if the file
* doesn't support non-blocking read/write attempts * doesn't support non-blocking read/write attempts
...@@ -6304,7 +6277,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, ...@@ -6304,7 +6277,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
{ {
struct io_submit_state *state; struct io_submit_state *state;
unsigned int sqe_flags; unsigned int sqe_flags;
int id, ret = 0; int ret = 0;
req->opcode = READ_ONCE(sqe->opcode); req->opcode = READ_ONCE(sqe->opcode);
/* same numerical values with corresponding REQ_F_*, safe to copy */ /* same numerical values with corresponding REQ_F_*, safe to copy */
...@@ -6336,15 +6309,9 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, ...@@ -6336,15 +6309,9 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
!io_op_defs[req->opcode].buffer_select) !io_op_defs[req->opcode].buffer_select)
return -EOPNOTSUPP; return -EOPNOTSUPP;
id = READ_ONCE(sqe->personality); req->work.list.next = NULL;
if (id) { req->work.flags = 0;
__io_req_init_async(req); req->work.personality = READ_ONCE(sqe->personality);
req->work.creds = idr_find(&ctx->personality_idr, id);
if (unlikely(!req->work.creds))
return -EINVAL;
get_cred(req->work.creds);
}
state = &ctx->submit_state; state = &ctx->submit_state;
/* /*
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment