Skip to content

L
linux
Commit 0702e536 authored by Jens Axboe's avatar Jens Axboe
Browse files
Options

io_uring: define a 'prep' and 'issue' handler for each opcode 

Rather than have two giant switches for doing request preparation and
then for doing request issue, add a prep and issue handler for each
of them in the io_op_defs[] request definition.
Signed-off-by: default avatarJens Axboe <axboe@kernel.dk>
parent e0dccc3b
Hide whitespace changes
Inline Side-by-side
Showing with 365 additions and 473 deletions
fs/io_uring.c
View file @ 0702e536
...@@ -1110,231 +1110,13 @@ struct io_op_def { ...@@ -1110,231 +1110,13 @@ struct io_op_def {
unsigned iopoll : 1; unsigned iopoll : 1;
/* size of async data needed, if any */ /* size of async data needed, if any */
unsigned short async_size; unsigned short async_size;
};
static const struct io_op_def io_op_defs[] = { int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);
[IORING_OP_NOP] = { int (*issue)(struct io_kiocb *, unsigned int);
.audit_skip = 1,
.iopoll = 1,
},
[IORING_OP_READV] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
.needs_async_setup = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
},
[IORING_OP_WRITEV] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_async_setup = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
},
[IORING_OP_FSYNC] = {
.needs_file = 1,
.audit_skip = 1,
},
[IORING_OP_READ_FIXED] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
},
[IORING_OP_WRITE_FIXED] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
},
[IORING_OP_POLL_ADD] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
},
[IORING_OP_POLL_REMOVE] = {
.audit_skip = 1,
},
[IORING_OP_SYNC_FILE_RANGE] = {
.needs_file = 1,
.audit_skip = 1,
},
[IORING_OP_SENDMSG] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_async_setup = 1,
.ioprio = 1,
.async_size = sizeof(struct io_async_msghdr),
},
[IORING_OP_RECVMSG] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
.needs_async_setup = 1,
.ioprio = 1,
.async_size = sizeof(struct io_async_msghdr),
},
[IORING_OP_TIMEOUT] = {
.audit_skip = 1,
.async_size = sizeof(struct io_timeout_data),
},
[IORING_OP_TIMEOUT_REMOVE] = {
/* used by timeout updates' prep() */
.audit_skip = 1,
},
[IORING_OP_ACCEPT] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.poll_exclusive = 1,
.ioprio = 1, /* used for flags */
},
[IORING_OP_ASYNC_CANCEL] = {
.audit_skip = 1,
},
[IORING_OP_LINK_TIMEOUT] = {
.audit_skip = 1,
.async_size = sizeof(struct io_timeout_data),
},
[IORING_OP_CONNECT] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_async_setup = 1,
.async_size = sizeof(struct io_async_connect),
},
[IORING_OP_FALLOCATE] = {
.needs_file = 1,
},
[IORING_OP_OPENAT] = {},
[IORING_OP_CLOSE] = {},
[IORING_OP_FILES_UPDATE] = {
.audit_skip = 1,
.iopoll = 1,
},
[IORING_OP_STATX] = {
.audit_skip = 1,
},
[IORING_OP_READ] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
},
[IORING_OP_WRITE] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
},
[IORING_OP_FADVISE] = {
.needs_file = 1,
.audit_skip = 1,
},
[IORING_OP_MADVISE] = {},
[IORING_OP_SEND] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.audit_skip = 1,
.ioprio = 1,
},
[IORING_OP_RECV] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
.audit_skip = 1,
.ioprio = 1,
},
[IORING_OP_OPENAT2] = {
},
[IORING_OP_EPOLL_CTL] = {
.unbound_nonreg_file = 1,
.audit_skip = 1,
},
[IORING_OP_SPLICE] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
},
[IORING_OP_PROVIDE_BUFFERS] = {
.audit_skip = 1,
.iopoll = 1,
},
[IORING_OP_REMOVE_BUFFERS] = {
.audit_skip = 1,
.iopoll = 1,
},
[IORING_OP_TEE] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
},
[IORING_OP_SHUTDOWN] = {
.needs_file = 1,
},
[IORING_OP_RENAMEAT] = {},
[IORING_OP_UNLINKAT] = {},
[IORING_OP_MKDIRAT] = {},
[IORING_OP_SYMLINKAT] = {},
[IORING_OP_LINKAT] = {},
[IORING_OP_MSG_RING] = {
.needs_file = 1,
.iopoll = 1,
},
[IORING_OP_FSETXATTR] = {
.needs_file = 1
},
[IORING_OP_SETXATTR] = {},
[IORING_OP_FGETXATTR] = {
.needs_file = 1
},
[IORING_OP_GETXATTR] = {},
[IORING_OP_SOCKET] = {
.audit_skip = 1,
},
[IORING_OP_URING_CMD] = {
.needs_file = 1,
.plug = 1,
.needs_async_setup = 1,
.async_size = uring_cmd_pdu_size(1),
},
}; };
static const struct io_op_def io_op_defs[];
/* requests with any of those set should undergo io_disarm_next() */ /* requests with any of those set should undergo io_disarm_next() */
#define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL) #define IO_DISARM_MASK (REQ_F_ARM_LTIMEOUT | REQ_F_LINK_TIMEOUT | REQ_F_FAIL)
#define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK) #define IO_REQ_LINK_FLAGS (REQ_F_LINK | REQ_F_HARDLINK)
...@@ -8039,96 +7821,33 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) ...@@ -8039,96 +7821,33 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags)
return 0; return 0;
} }
static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int io_req_prep_async(struct io_kiocb *req)
{ {
const struct io_op_def *def = &io_op_defs[req->opcode];
/* assign early for deferred execution for non-fixed file */
if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE))
req->file = io_file_get_normal(req, req->cqe.fd);
if (!def->needs_async_setup)
return 0;
if (WARN_ON_ONCE(req_has_async_data(req)))
return -EFAULT;
if (io_alloc_async_data(req))
return -EAGAIN;
switch (req->opcode) { switch (req->opcode) {
case IORING_OP_NOP:
return io_nop_prep(req, sqe);
case IORING_OP_READV: case IORING_OP_READV:
case IORING_OP_READ_FIXED: return io_readv_prep_async(req);
case IORING_OP_READ:
case IORING_OP_WRITEV: case IORING_OP_WRITEV:
case IORING_OP_WRITE_FIXED: return io_writev_prep_async(req);
case IORING_OP_WRITE:
return io_prep_rw(req, sqe);
case IORING_OP_POLL_ADD:
return io_poll_add_prep(req, sqe);
case IORING_OP_POLL_REMOVE:
return io_poll_remove_prep(req, sqe);
case IORING_OP_FSYNC:
return io_fsync_prep(req, sqe);
case IORING_OP_SYNC_FILE_RANGE:
return io_sfr_prep(req, sqe);
case IORING_OP_SENDMSG: case IORING_OP_SENDMSG:
case IORING_OP_SEND: return io_sendmsg_prep_async(req);
return io_sendmsg_prep(req, sqe);
case IORING_OP_RECVMSG: case IORING_OP_RECVMSG:
case IORING_OP_RECV: return io_recvmsg_prep_async(req);
return io_recvmsg_prep(req, sqe);
case IORING_OP_CONNECT: case IORING_OP_CONNECT:
return io_connect_prep(req, sqe); return io_connect_prep_async(req);
case IORING_OP_TIMEOUT:
return io_timeout_prep(req, sqe);
case IORING_OP_TIMEOUT_REMOVE:
return io_timeout_remove_prep(req, sqe);
case IORING_OP_ASYNC_CANCEL:
return io_async_cancel_prep(req, sqe);
case IORING_OP_LINK_TIMEOUT:
return io_link_timeout_prep(req, sqe);
case IORING_OP_ACCEPT:
return io_accept_prep(req, sqe);
case IORING_OP_FALLOCATE:
return io_fallocate_prep(req, sqe);
case IORING_OP_OPENAT:
return io_openat_prep(req, sqe);
case IORING_OP_CLOSE:
return io_close_prep(req, sqe);
case IORING_OP_FILES_UPDATE:
return io_files_update_prep(req, sqe);
case IORING_OP_STATX:
return io_statx_prep(req, sqe);
case IORING_OP_FADVISE:
return io_fadvise_prep(req, sqe);
case IORING_OP_MADVISE:
return io_madvise_prep(req, sqe);
case IORING_OP_OPENAT2:
return io_openat2_prep(req, sqe);
case IORING_OP_EPOLL_CTL:
return io_epoll_ctl_prep(req, sqe);
case IORING_OP_SPLICE:
return io_splice_prep(req, sqe);
case IORING_OP_PROVIDE_BUFFERS:
return io_provide_buffers_prep(req, sqe);
case IORING_OP_REMOVE_BUFFERS:
return io_remove_buffers_prep(req, sqe);
case IORING_OP_TEE:
return io_tee_prep(req, sqe);
case IORING_OP_SHUTDOWN:
return io_shutdown_prep(req, sqe);
case IORING_OP_RENAMEAT:
return io_renameat_prep(req, sqe);
case IORING_OP_UNLINKAT:
return io_unlinkat_prep(req, sqe);
case IORING_OP_MKDIRAT:
return io_mkdirat_prep(req, sqe);
case IORING_OP_SYMLINKAT:
return io_symlinkat_prep(req, sqe);
case IORING_OP_LINKAT:
return io_linkat_prep(req, sqe);
case IORING_OP_MSG_RING:
return io_msg_ring_prep(req, sqe);
case IORING_OP_FSETXATTR:
return io_fsetxattr_prep(req, sqe);
case IORING_OP_SETXATTR:
return io_setxattr_prep(req, sqe);
case IORING_OP_FGETXATTR:
return io_fgetxattr_prep(req, sqe);
case IORING_OP_GETXATTR:
return io_getxattr_prep(req, sqe);
case IORING_OP_SOCKET:
return io_socket_prep(req, sqe);
case IORING_OP_URING_CMD: case IORING_OP_URING_CMD:
return io_uring_cmd_prep(req, sqe); return io_uring_cmd_prep_async(req);
} }
printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n", printk_once(KERN_WARNING "io_uring: unhandled opcode %d\n",
...@@ -8136,39 +7855,6 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) ...@@ -8136,39 +7855,6 @@ static int io_req_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return -EINVAL; return -EINVAL;
} }
static int io_req_prep_async(struct io_kiocb *req)
{
const struct io_op_def *def = &io_op_defs[req->opcode];
/* assign early for deferred execution for non-fixed file */
if (def->needs_file && !(req->flags & REQ_F_FIXED_FILE))
req->file = io_file_get_normal(req, req->cqe.fd);
if (!def->needs_async_setup)
return 0;
if (WARN_ON_ONCE(req_has_async_data(req)))
return -EFAULT;
if (io_alloc_async_data(req))
return -EAGAIN;
switch (req->opcode) {
case IORING_OP_READV:
return io_readv_prep_async(req);
case IORING_OP_WRITEV:
return io_writev_prep_async(req);
case IORING_OP_SENDMSG:
return io_sendmsg_prep_async(req);
case IORING_OP_RECVMSG:
return io_recvmsg_prep_async(req);
case IORING_OP_CONNECT:
return io_connect_prep_async(req);
case IORING_OP_URING_CMD:
return io_uring_cmd_prep_async(req);
}
printk_once(KERN_WARNING "io_uring: prep_async() bad opcode %d\n",
req->opcode);
return -EFAULT;
}
static u32 io_get_sequence(struct io_kiocb *req) static u32 io_get_sequence(struct io_kiocb *req)
{ {
u32 seq = req->ctx->cached_sq_head; u32 seq = req->ctx->cached_sq_head;
...@@ -8335,141 +8021,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags) ...@@ -8335,141 +8021,7 @@ static int io_issue_sqe(struct io_kiocb *req, unsigned int issue_flags)
if (!def->audit_skip) if (!def->audit_skip)
audit_uring_entry(req->opcode); audit_uring_entry(req->opcode);
switch (req->opcode) { ret = def->issue(req, issue_flags);
case IORING_OP_NOP:
ret = io_nop(req, issue_flags);
break;
case IORING_OP_READV:
case IORING_OP_READ_FIXED:
case IORING_OP_READ:
ret = io_read(req, issue_flags);
break;
case IORING_OP_WRITEV:
case IORING_OP_WRITE_FIXED:
case IORING_OP_WRITE:
ret = io_write(req, issue_flags);
break;
case IORING_OP_FSYNC:
ret = io_fsync(req, issue_flags);
break;
case IORING_OP_POLL_ADD:
ret = io_poll_add(req, issue_flags);
break;
case IORING_OP_POLL_REMOVE:
ret = io_poll_remove(req, issue_flags);
break;
case IORING_OP_SYNC_FILE_RANGE:
ret = io_sync_file_range(req, issue_flags);
break;
case IORING_OP_SENDMSG:
ret = io_sendmsg(req, issue_flags);
break;
case IORING_OP_SEND:
ret = io_send(req, issue_flags);
break;
case IORING_OP_RECVMSG:
ret = io_recvmsg(req, issue_flags);
break;
case IORING_OP_RECV:
ret = io_recv(req, issue_flags);
break;
case IORING_OP_TIMEOUT:
ret = io_timeout(req, issue_flags);
break;
case IORING_OP_TIMEOUT_REMOVE:
ret = io_timeout_remove(req, issue_flags);
break;
case IORING_OP_ACCEPT:
ret = io_accept(req, issue_flags);
break;
case IORING_OP_CONNECT:
ret = io_connect(req, issue_flags);
break;
case IORING_OP_ASYNC_CANCEL:
ret = io_async_cancel(req, issue_flags);
break;
case IORING_OP_FALLOCATE:
ret = io_fallocate(req, issue_flags);
break;
case IORING_OP_OPENAT:
ret = io_openat(req, issue_flags);
break;
case IORING_OP_CLOSE:
ret = io_close(req, issue_flags);
break;
case IORING_OP_FILES_UPDATE:
ret = io_files_update(req, issue_flags);
break;
case IORING_OP_STATX:
ret = io_statx(req, issue_flags);
break;
case IORING_OP_FADVISE:
ret = io_fadvise(req, issue_flags);
break;
case IORING_OP_MADVISE:
ret = io_madvise(req, issue_flags);
break;
case IORING_OP_OPENAT2:
ret = io_openat2(req, issue_flags);
break;
case IORING_OP_EPOLL_CTL:
ret = io_epoll_ctl(req, issue_flags);
break;
case IORING_OP_SPLICE:
ret = io_splice(req, issue_flags);
break;
case IORING_OP_PROVIDE_BUFFERS:
ret = io_provide_buffers(req, issue_flags);
break;
case IORING_OP_REMOVE_BUFFERS:
ret = io_remove_buffers(req, issue_flags);
break;
case IORING_OP_TEE:
ret = io_tee(req, issue_flags);
break;
case IORING_OP_SHUTDOWN:
ret = io_shutdown(req, issue_flags);
break;
case IORING_OP_RENAMEAT:
ret = io_renameat(req, issue_flags);
break;
case IORING_OP_UNLINKAT:
ret = io_unlinkat(req, issue_flags);
break;
case IORING_OP_MKDIRAT:
ret = io_mkdirat(req, issue_flags);
break;
case IORING_OP_SYMLINKAT:
ret = io_symlinkat(req, issue_flags);
break;
case IORING_OP_LINKAT:
ret = io_linkat(req, issue_flags);
break;
case IORING_OP_MSG_RING:
ret = io_msg_ring(req, issue_flags);
break;
case IORING_OP_FSETXATTR:
ret = io_fsetxattr(req, issue_flags);
break;
case IORING_OP_SETXATTR:
ret = io_setxattr(req, issue_flags);
break;
case IORING_OP_FGETXATTR:
ret = io_fgetxattr(req, issue_flags);
break;
case IORING_OP_GETXATTR:
ret = io_getxattr(req, issue_flags);
break;
case IORING_OP_SOCKET:
ret = io_socket(req, issue_flags);
break;
case IORING_OP_URING_CMD:
ret = io_uring_cmd(req, issue_flags);
break;
default:
ret = -EINVAL;
break;
}
if (!def->audit_skip) if (!def->audit_skip)
audit_uring_exit(!ret, ret); audit_uring_exit(!ret, ret);
...@@ -8898,7 +8450,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, ...@@ -8898,7 +8450,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
req->flags |= REQ_F_CREDS; req->flags |= REQ_F_CREDS;
} }
return io_req_prep(req, sqe); return def->prep(req, sqe);
} }
static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe, static __cold int io_submit_fail_init(const struct io_uring_sqe *sqe,
...@@ -13200,8 +12752,343 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, ...@@ -13200,8 +12752,343 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
return ret; return ret;
} }
static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
{
WARN_ON_ONCE(1);
return -ECANCELED;
}
static const struct io_op_def io_op_defs[] = {
[IORING_OP_NOP] = {
.audit_skip = 1,
.iopoll = 1,
.prep = io_nop_prep,
.issue = io_nop,
},
[IORING_OP_READV] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
.needs_async_setup = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
.prep = io_prep_rw,
.issue = io_read,
},
[IORING_OP_WRITEV] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_async_setup = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
.prep = io_prep_rw,
.issue = io_write,
},
[IORING_OP_FSYNC] = {
.needs_file = 1,
.audit_skip = 1,
.prep = io_fsync_prep,
.issue = io_fsync,
},
[IORING_OP_READ_FIXED] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
.prep = io_prep_rw,
.issue = io_read,
},
[IORING_OP_WRITE_FIXED] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
.prep = io_prep_rw,
.issue = io_write,
},
[IORING_OP_POLL_ADD] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
.prep = io_poll_add_prep,
.issue = io_poll_add,
},
[IORING_OP_POLL_REMOVE] = {
.audit_skip = 1,
.prep = io_poll_remove_prep,
.issue = io_poll_remove,
},
[IORING_OP_SYNC_FILE_RANGE] = {
.needs_file = 1,
.audit_skip = 1,
.prep = io_sfr_prep,
.issue = io_sync_file_range,
},
[IORING_OP_SENDMSG] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_async_setup = 1,
.ioprio = 1,
.async_size = sizeof(struct io_async_msghdr),
.prep = io_sendmsg_prep,
.issue = io_sendmsg,
},
[IORING_OP_RECVMSG] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
.needs_async_setup = 1,
.ioprio = 1,
.async_size = sizeof(struct io_async_msghdr),
.prep = io_recvmsg_prep,
.issue = io_recvmsg,
},
[IORING_OP_TIMEOUT] = {
.audit_skip = 1,
.async_size = sizeof(struct io_timeout_data),
.prep = io_timeout_prep,
.issue = io_timeout,
},
[IORING_OP_TIMEOUT_REMOVE] = {
/* used by timeout updates' prep() */
.audit_skip = 1,
.prep = io_timeout_remove_prep,
.issue = io_timeout_remove,
},
[IORING_OP_ACCEPT] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.poll_exclusive = 1,
.ioprio = 1, /* used for flags */
.prep = io_accept_prep,
.issue = io_accept,
},
[IORING_OP_ASYNC_CANCEL] = {
.audit_skip = 1,
.prep = io_async_cancel_prep,
.issue = io_async_cancel,
},
[IORING_OP_LINK_TIMEOUT] = {
.audit_skip = 1,
.async_size = sizeof(struct io_timeout_data),
.prep = io_link_timeout_prep,
.issue = io_no_issue,
},
[IORING_OP_CONNECT] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.needs_async_setup = 1,
.async_size = sizeof(struct io_async_connect),
.prep = io_connect_prep,
.issue = io_connect,
},
[IORING_OP_FALLOCATE] = {
.needs_file = 1,
.prep = io_fallocate_prep,
.issue = io_fallocate,
},
[IORING_OP_OPENAT] = {
.prep = io_openat_prep,
.issue = io_openat,
},
[IORING_OP_CLOSE] = {
.prep = io_close_prep,
.issue = io_close,
},
[IORING_OP_FILES_UPDATE] = {
.audit_skip = 1,
.iopoll = 1,
.prep = io_files_update_prep,
.issue = io_files_update,
},
[IORING_OP_STATX] = {
.audit_skip = 1,
.prep = io_statx_prep,
.issue = io_statx,
},
[IORING_OP_READ] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
.prep = io_prep_rw,
.issue = io_read,
},
[IORING_OP_WRITE] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.plug = 1,
.audit_skip = 1,
.ioprio = 1,
.iopoll = 1,
.async_size = sizeof(struct io_async_rw),
.prep = io_prep_rw,
.issue = io_write,
},
[IORING_OP_FADVISE] = {
.needs_file = 1,
.audit_skip = 1,
.prep = io_fadvise_prep,
.issue = io_fadvise,
},
[IORING_OP_MADVISE] = {
.prep = io_madvise_prep,
.issue = io_madvise,
},
[IORING_OP_SEND] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollout = 1,
.audit_skip = 1,
.ioprio = 1,
.prep = io_sendmsg_prep,
.issue = io_send,
},
[IORING_OP_RECV] = {
.needs_file = 1,
.unbound_nonreg_file = 1,
.pollin = 1,
.buffer_select = 1,
.audit_skip = 1,
.ioprio = 1,
.prep = io_recvmsg_prep,
.issue = io_recv,
},
[IORING_OP_OPENAT2] = {
.prep = io_openat2_prep,
.issue = io_openat2,
},
[IORING_OP_EPOLL_CTL] = {
.unbound_nonreg_file = 1,
.audit_skip = 1,
.prep = io_epoll_ctl_prep,
.issue = io_epoll_ctl,
},
[IORING_OP_SPLICE] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
.prep = io_splice_prep,
.issue = io_splice,
},
[IORING_OP_PROVIDE_BUFFERS] = {
.audit_skip = 1,
.iopoll = 1,
.prep = io_provide_buffers_prep,
.issue = io_provide_buffers,
},
[IORING_OP_REMOVE_BUFFERS] = {
.audit_skip = 1,
.iopoll = 1,
.prep = io_remove_buffers_prep,
.issue = io_remove_buffers,
},
[IORING_OP_TEE] = {
.needs_file = 1,
.hash_reg_file = 1,
.unbound_nonreg_file = 1,
.audit_skip = 1,
.prep = io_tee_prep,
.issue = io_tee,
},
[IORING_OP_SHUTDOWN] = {
.needs_file = 1,
.prep = io_shutdown_prep,
.issue = io_shutdown,
},
[IORING_OP_RENAMEAT] = {
.prep = io_renameat_prep,
.issue = io_renameat,
},
[IORING_OP_UNLINKAT] = {
.prep = io_unlinkat_prep,
.issue = io_unlinkat,
},
[IORING_OP_MKDIRAT] = {
.prep = io_mkdirat_prep,
.issue = io_mkdirat,
},
[IORING_OP_SYMLINKAT] = {
.prep = io_symlinkat_prep,
.issue = io_symlinkat,
},
[IORING_OP_LINKAT] = {
.prep = io_linkat_prep,
.issue = io_linkat,
},
[IORING_OP_MSG_RING] = {
.needs_file = 1,
.iopoll = 1,
.prep = io_msg_ring_prep,
.issue = io_msg_ring,
},
[IORING_OP_FSETXATTR] = {
.needs_file = 1,
.prep = io_fsetxattr_prep,
.issue = io_fsetxattr,
},
[IORING_OP_SETXATTR] = {
.prep = io_setxattr_prep,
.issue = io_setxattr,
},
[IORING_OP_FGETXATTR] = {
.needs_file = 1,
.prep = io_fgetxattr_prep,
.issue = io_fgetxattr,
},
[IORING_OP_GETXATTR] = {
.prep = io_getxattr_prep,
.issue = io_getxattr,
},
[IORING_OP_SOCKET] = {
.audit_skip = 1,
.prep = io_socket_prep,
.issue = io_socket,
},
[IORING_OP_URING_CMD] = {
.needs_file = 1,
.plug = 1,
.needs_async_setup = 1,
.async_size = uring_cmd_pdu_size(1),
.prep = io_uring_cmd_prep,
.issue = io_uring_cmd,
},
};
static int __init io_uring_init(void) static int __init io_uring_init(void)
{ {
int i;
#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \ #define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \
BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \ BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \
BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \ BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \
...@@ -13266,6 +13153,11 @@ static int __init io_uring_init(void) ...@@ -13266,6 +13153,11 @@ static int __init io_uring_init(void)
BUILD_BUG_ON(sizeof(struct io_uring_cmd) > 64); BUILD_BUG_ON(sizeof(struct io_uring_cmd) > 64);
for (i = 0; i < ARRAY_SIZE(io_op_defs); i++) {
BUG_ON(!io_op_defs[i].prep);
BUG_ON(!io_op_defs[i].issue);
}
req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC | req_cachep = KMEM_CACHE(io_kiocb, SLAB_HWCACHE_ALIGN | SLAB_PANIC |
SLAB_ACCOUNT); SLAB_ACCOUNT);
return 0; return 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7