Commit 009a82f6 authored by Trond Myklebust's avatar Trond Myklebust

SUNRPC: Micro-optimise when the task is known not to be sleeping

In cases where we know the task is not sleeping, try to optimise
away the indirect call to task->tk_action() by replacing it with
a direct call.
Only change tail calls, to allow gcc to perform tail call
elimination.
Signed-off-by: default avatarTrond Myklebust <trond.myklebust@hammerspace.com>
parent 03e51d32
...@@ -304,4 +304,12 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt) ...@@ -304,4 +304,12 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt)
} }
#endif /* CONFIG_SUNRPC_SWAP */ #endif /* CONFIG_SUNRPC_SWAP */
static inline bool
rpc_task_need_resched(const struct rpc_task *task)
{
if (RPC_IS_QUEUED(task) || task->tk_callback)
return true;
return false;
}
#endif /* _LINUX_SUNRPC_SCHED_H_ */ #endif /* _LINUX_SUNRPC_SCHED_H_ */
...@@ -1540,6 +1540,7 @@ call_start(struct rpc_task *task) ...@@ -1540,6 +1540,7 @@ call_start(struct rpc_task *task)
clnt->cl_stats->rpccnt++; clnt->cl_stats->rpccnt++;
task->tk_action = call_reserve; task->tk_action = call_reserve;
rpc_task_set_transport(task, clnt); rpc_task_set_transport(task, clnt);
call_reserve(task);
} }
/* /*
...@@ -1553,6 +1554,9 @@ call_reserve(struct rpc_task *task) ...@@ -1553,6 +1554,9 @@ call_reserve(struct rpc_task *task)
task->tk_status = 0; task->tk_status = 0;
task->tk_action = call_reserveresult; task->tk_action = call_reserveresult;
xprt_reserve(task); xprt_reserve(task);
if (rpc_task_need_resched(task))
return;
call_reserveresult(task);
} }
static void call_retry_reserve(struct rpc_task *task); static void call_retry_reserve(struct rpc_task *task);
...@@ -1575,6 +1579,7 @@ call_reserveresult(struct rpc_task *task) ...@@ -1575,6 +1579,7 @@ call_reserveresult(struct rpc_task *task)
if (status >= 0) { if (status >= 0) {
if (task->tk_rqstp) { if (task->tk_rqstp) {
task->tk_action = call_refresh; task->tk_action = call_refresh;
call_refresh(task);
return; return;
} }
...@@ -1600,6 +1605,7 @@ call_reserveresult(struct rpc_task *task) ...@@ -1600,6 +1605,7 @@ call_reserveresult(struct rpc_task *task)
/* fall through */ /* fall through */
case -EAGAIN: /* woken up; retry */ case -EAGAIN: /* woken up; retry */
task->tk_action = call_retry_reserve; task->tk_action = call_retry_reserve;
call_retry_reserve(task);
return; return;
case -EIO: /* probably a shutdown */ case -EIO: /* probably a shutdown */
break; break;
...@@ -1622,6 +1628,9 @@ call_retry_reserve(struct rpc_task *task) ...@@ -1622,6 +1628,9 @@ call_retry_reserve(struct rpc_task *task)
task->tk_status = 0; task->tk_status = 0;
task->tk_action = call_reserveresult; task->tk_action = call_reserveresult;
xprt_retry_reserve(task); xprt_retry_reserve(task);
if (rpc_task_need_resched(task))
return;
call_reserveresult(task);
} }
/* /*
...@@ -1636,6 +1645,9 @@ call_refresh(struct rpc_task *task) ...@@ -1636,6 +1645,9 @@ call_refresh(struct rpc_task *task)
task->tk_status = 0; task->tk_status = 0;
task->tk_client->cl_stats->rpcauthrefresh++; task->tk_client->cl_stats->rpcauthrefresh++;
rpcauth_refreshcred(task); rpcauth_refreshcred(task);
if (rpc_task_need_resched(task))
return;
call_refreshresult(task);
} }
/* /*
...@@ -1654,6 +1666,7 @@ call_refreshresult(struct rpc_task *task) ...@@ -1654,6 +1666,7 @@ call_refreshresult(struct rpc_task *task)
case 0: case 0:
if (rpcauth_uptodatecred(task)) { if (rpcauth_uptodatecred(task)) {
task->tk_action = call_allocate; task->tk_action = call_allocate;
call_allocate(task);
return; return;
} }
/* Use rate-limiting and a max number of retries if refresh /* Use rate-limiting and a max number of retries if refresh
...@@ -1672,6 +1685,7 @@ call_refreshresult(struct rpc_task *task) ...@@ -1672,6 +1685,7 @@ call_refreshresult(struct rpc_task *task)
task->tk_cred_retry--; task->tk_cred_retry--;
dprintk("RPC: %5u %s: retry refresh creds\n", dprintk("RPC: %5u %s: retry refresh creds\n",
task->tk_pid, __func__); task->tk_pid, __func__);
call_refresh(task);
return; return;
} }
dprintk("RPC: %5u %s: refresh creds failed with error %d\n", dprintk("RPC: %5u %s: refresh creds failed with error %d\n",
...@@ -1697,8 +1711,10 @@ call_allocate(struct rpc_task *task) ...@@ -1697,8 +1711,10 @@ call_allocate(struct rpc_task *task)
task->tk_status = 0; task->tk_status = 0;
task->tk_action = call_encode; task->tk_action = call_encode;
if (req->rq_buffer) if (req->rq_buffer) {
call_encode(task);
return; return;
}
if (proc->p_proc != 0) { if (proc->p_proc != 0) {
BUG_ON(proc->p_arglen == 0); BUG_ON(proc->p_arglen == 0);
...@@ -1719,8 +1735,12 @@ call_allocate(struct rpc_task *task) ...@@ -1719,8 +1735,12 @@ call_allocate(struct rpc_task *task)
status = xprt->ops->buf_alloc(task); status = xprt->ops->buf_alloc(task);
xprt_inject_disconnect(xprt); xprt_inject_disconnect(xprt);
if (status == 0) if (status == 0) {
if (rpc_task_need_resched(task))
return; return;
call_encode(task);
return;
}
if (status != -ENOMEM) { if (status != -ENOMEM) {
rpc_exit(task, status); rpc_exit(task, status);
return; return;
...@@ -1803,12 +1823,8 @@ call_encode(struct rpc_task *task) ...@@ -1803,12 +1823,8 @@ call_encode(struct rpc_task *task)
xprt_request_enqueue_receive(task); xprt_request_enqueue_receive(task);
xprt_request_enqueue_transmit(task); xprt_request_enqueue_transmit(task);
out: out:
task->tk_action = call_transmit;
/* Check that the connection is OK */
if (!xprt_bound(task->tk_xprt))
task->tk_action = call_bind; task->tk_action = call_bind;
else if (!xprt_connected(task->tk_xprt)) call_bind(task);
task->tk_action = call_connect;
} }
/* /*
...@@ -1842,14 +1858,17 @@ call_bind(struct rpc_task *task) ...@@ -1842,14 +1858,17 @@ call_bind(struct rpc_task *task)
return; return;
} }
if (xprt_bound(xprt)) {
task->tk_action = call_connect;
call_connect(task);
return;
}
dprint_status(task); dprint_status(task);
task->tk_action = call_connect;
if (!xprt_bound(xprt)) {
task->tk_action = call_bind_status; task->tk_action = call_bind_status;
task->tk_timeout = xprt->bind_timeout; task->tk_timeout = xprt->bind_timeout;
xprt->ops->rpcbind(task); xprt->ops->rpcbind(task);
}
} }
/* /*
...@@ -1869,6 +1888,7 @@ call_bind_status(struct rpc_task *task) ...@@ -1869,6 +1888,7 @@ call_bind_status(struct rpc_task *task)
dprint_status(task); dprint_status(task);
task->tk_status = 0; task->tk_status = 0;
task->tk_action = call_connect; task->tk_action = call_connect;
call_connect(task);
return; return;
} }
...@@ -1949,12 +1969,16 @@ call_connect(struct rpc_task *task) ...@@ -1949,12 +1969,16 @@ call_connect(struct rpc_task *task)
return; return;
} }
if (xprt_connected(xprt)) {
task->tk_action = call_transmit;
call_transmit(task);
return;
}
dprintk("RPC: %5u call_connect xprt %p %s connected\n", dprintk("RPC: %5u call_connect xprt %p %s connected\n",
task->tk_pid, xprt, task->tk_pid, xprt,
(xprt_connected(xprt) ? "is" : "is not")); (xprt_connected(xprt) ? "is" : "is not"));
task->tk_action = call_transmit;
if (!xprt_connected(xprt)) {
task->tk_action = call_connect_status; task->tk_action = call_connect_status;
if (task->tk_status < 0) if (task->tk_status < 0)
return; return;
...@@ -1963,7 +1987,6 @@ call_connect(struct rpc_task *task) ...@@ -1963,7 +1987,6 @@ call_connect(struct rpc_task *task)
return; return;
} }
xprt_connect(task); xprt_connect(task);
}
} }
/* /*
...@@ -2016,6 +2039,7 @@ call_connect_status(struct rpc_task *task) ...@@ -2016,6 +2039,7 @@ call_connect_status(struct rpc_task *task)
case 0: case 0:
clnt->cl_stats->netreconn++; clnt->cl_stats->netreconn++;
task->tk_action = call_transmit; task->tk_action = call_transmit;
call_transmit(task);
return; return;
} }
rpc_exit(task, status); rpc_exit(task, status);
...@@ -2040,7 +2064,6 @@ call_transmit(struct rpc_task *task) ...@@ -2040,7 +2064,6 @@ call_transmit(struct rpc_task *task)
dprint_status(task); dprint_status(task);
task->tk_action = call_transmit_status; task->tk_action = call_transmit_status;
if (test_bit(RPC_TASK_NEED_XMIT, &task->tk_runstate)) {
if (!xprt_prepare_transmit(task)) if (!xprt_prepare_transmit(task))
return; return;
task->tk_status = 0; task->tk_status = 0;
...@@ -2051,8 +2074,10 @@ call_transmit(struct rpc_task *task) ...@@ -2051,8 +2074,10 @@ call_transmit(struct rpc_task *task)
} }
xprt_transmit(task); xprt_transmit(task);
} }
}
xprt_end_transmit(task); xprt_end_transmit(task);
if (rpc_task_need_resched(task))
return;
call_transmit_status(task);
} }
/* /*
...@@ -2067,8 +2092,12 @@ call_transmit_status(struct rpc_task *task) ...@@ -2067,8 +2092,12 @@ call_transmit_status(struct rpc_task *task)
* Common case: success. Force the compiler to put this * Common case: success. Force the compiler to put this
* test first. * test first.
*/ */
if (task->tk_status == 0) { if (rpc_task_transmitted(task)) {
if (task->tk_status == 0)
xprt_request_wait_receive(task); xprt_request_wait_receive(task);
if (rpc_task_need_resched(task))
return;
call_status(task);
return; return;
} }
...@@ -2129,6 +2158,7 @@ call_bc_encode(struct rpc_task *task) ...@@ -2129,6 +2158,7 @@ call_bc_encode(struct rpc_task *task)
{ {
xprt_request_enqueue_transmit(task); xprt_request_enqueue_transmit(task);
task->tk_action = call_bc_transmit; task->tk_action = call_bc_transmit;
call_bc_transmit(task);
} }
/* /*
...@@ -2219,6 +2249,7 @@ call_status(struct rpc_task *task) ...@@ -2219,6 +2249,7 @@ call_status(struct rpc_task *task)
status = task->tk_status; status = task->tk_status;
if (status >= 0) { if (status >= 0) {
task->tk_action = call_decode; task->tk_action = call_decode;
call_decode(task);
return; return;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment