Commit a1bf4c7d authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'nfs-for-4.17-1' of git://git.linux-nfs.org/projects/anna/linux-nfs

Pull NFS client updates from Anna Schumaker:
 "Stable bugfixes:
   - xprtrdma: Fix corner cases when handling device removal # v4.12+
   - xprtrdma: Fix latency regression on NUMA NFS/RDMA clients # v4.15+

  Features:
   - New sunrpc tracepoint for RPC pings
   - Finer grained NFSv4 attribute checking
   - Don't unnecessarily return NFS v4 delegations

  Other bugfixes and cleanups:
   - Several other small NFSoRDMA cleanups
   - Improvements to the sunrpc RTT measurements
   - A few sunrpc tracepoint cleanups
   - Various fixes for NFS v4 lock notifications
   - Various sunrpc and NFS v4 XDR encoding cleanups
   - Switch to the ida_simple API
   - Fix NFSv4.1 exclusive create
   - Forget acl cache after setattr operation
   - Don't advance the nfs_entry readdir cookie if xdr decoding fails"

* tag 'nfs-for-4.17-1' of git://git.linux-nfs.org/projects/anna/linux-nfs: (47 commits)
  NFS: advance nfs_entry cookie only after decoding completes successfully
  NFSv3/acl: forget acl cache after setattr
  NFSv4.1: Fix exclusive create
  NFSv4: Declare the size up to date after it was set.
  nfs: Use ida_simple API
  NFSv4: Fix the nfs_inode_set_delegation() arguments
  NFSv4: Clean up CB_GETATTR encoding
  NFSv4: Don't ask for attributes when ACCESS is protected by a delegation
  NFSv4: Add a helper to encode/decode struct timespec
  NFSv4: Clean up encode_attrs
  NFSv4; Clean up XDR encoding of type bitmap4
  NFSv4: Allow GFP_NOIO sleeps in decode_attr_owner/decode_attr_group
  SUNRPC: Add a helper for encoding opaque data inline
  SUNRPC: Add helpers for decoding opaque and string types
  NFSv4: Ignore change attribute invalidations if we hold a delegation
  NFS: More fine grained attribute tracking
  NFS: Don't force unnecessary cache invalidation in nfs_update_inode()
  NFS: Don't redirty the attribute cache in nfs_wcc_update_inode()
  NFS: Don't force a revalidation of all attributes if change is missing
  NFS: Convert NFS_INO_INVALID flags to unsigned long
  ...
parents 7214dd4e 98de9ce6
......@@ -535,35 +535,10 @@ static __be32 encode_string(struct xdr_stream *xdr, unsigned int len, const char
return 0;
}
#define CB_SUPPORTED_ATTR0 (FATTR4_WORD0_CHANGE|FATTR4_WORD0_SIZE)
#define CB_SUPPORTED_ATTR1 (FATTR4_WORD1_TIME_METADATA|FATTR4_WORD1_TIME_MODIFY)
static __be32 encode_attr_bitmap(struct xdr_stream *xdr, const uint32_t *bitmap, __be32 **savep)
static __be32 encode_attr_bitmap(struct xdr_stream *xdr, const uint32_t *bitmap, size_t sz)
{
__be32 bm[2];
__be32 *p;
bm[0] = htonl(bitmap[0] & CB_SUPPORTED_ATTR0);
bm[1] = htonl(bitmap[1] & CB_SUPPORTED_ATTR1);
if (bm[1] != 0) {
p = xdr_reserve_space(xdr, 16);
if (unlikely(p == NULL))
return htonl(NFS4ERR_RESOURCE);
*p++ = htonl(2);
*p++ = bm[0];
*p++ = bm[1];
} else if (bm[0] != 0) {
p = xdr_reserve_space(xdr, 12);
if (unlikely(p == NULL))
return htonl(NFS4ERR_RESOURCE);
*p++ = htonl(1);
*p++ = bm[0];
} else {
p = xdr_reserve_space(xdr, 8);
if (unlikely(p == NULL))
return htonl(NFS4ERR_RESOURCE);
*p++ = htonl(0);
}
*savep = p;
if (xdr_stream_encode_uint32_array(xdr, bitmap, sz) < 0)
return cpu_to_be32(NFS4ERR_RESOURCE);
return 0;
}
......@@ -656,9 +631,13 @@ static __be32 encode_getattr_res(struct svc_rqst *rqstp, struct xdr_stream *xdr,
if (unlikely(status != 0))
goto out;
status = encode_attr_bitmap(xdr, res->bitmap, &savep);
status = encode_attr_bitmap(xdr, res->bitmap, ARRAY_SIZE(res->bitmap));
if (unlikely(status != 0))
goto out;
status = cpu_to_be32(NFS4ERR_RESOURCE);
savep = xdr_reserve_space(xdr, sizeof(*savep));
if (unlikely(!savep))
goto out;
status = encode_attr_change(xdr, res->bitmap, res->change_attr);
if (unlikely(status != 0))
goto out;
......
......@@ -19,6 +19,7 @@
#include <linux/nfs_xdr.h>
#include "nfs4_fs.h"
#include "nfs4session.h"
#include "delegation.h"
#include "internal.h"
#include "nfs4trace.h"
......@@ -171,11 +172,15 @@ static int nfs_delegation_claim_opens(struct inode *inode,
* nfs_inode_reclaim_delegation - process a delegation reclaim request
* @inode: inode to process
* @cred: credential to use for request
* @res: new delegation state from server
* @type: delegation type
* @stateid: delegation stateid
* @pagemod_limit: write delegation "space_limit"
*
*/
void nfs_inode_reclaim_delegation(struct inode *inode, struct rpc_cred *cred,
struct nfs_openres *res)
fmode_t type,
const nfs4_stateid *stateid,
unsigned long pagemod_limit)
{
struct nfs_delegation *delegation;
struct rpc_cred *oldcred = NULL;
......@@ -185,9 +190,9 @@ void nfs_inode_reclaim_delegation(struct inode *inode, struct rpc_cred *cred,
if (delegation != NULL) {
spin_lock(&delegation->lock);
if (delegation->inode != NULL) {
nfs4_stateid_copy(&delegation->stateid, &res->delegation);
delegation->type = res->delegation_type;
delegation->pagemod_limit = res->pagemod_limit;
nfs4_stateid_copy(&delegation->stateid, stateid);
delegation->type = type;
delegation->pagemod_limit = pagemod_limit;
oldcred = delegation->cred;
delegation->cred = get_rpccred(cred);
clear_bit(NFS_DELEGATION_NEED_RECLAIM,
......@@ -195,14 +200,14 @@ void nfs_inode_reclaim_delegation(struct inode *inode, struct rpc_cred *cred,
spin_unlock(&delegation->lock);
rcu_read_unlock();
put_rpccred(oldcred);
trace_nfs4_reclaim_delegation(inode, res->delegation_type);
trace_nfs4_reclaim_delegation(inode, type);
return;
}
/* We appear to have raced with a delegation return. */
spin_unlock(&delegation->lock);
}
rcu_read_unlock();
nfs_inode_set_delegation(inode, cred, res);
nfs_inode_set_delegation(inode, cred, type, stateid, pagemod_limit);
}
static int nfs_do_return_delegation(struct inode *inode, struct nfs_delegation *delegation, int issync)
......@@ -329,11 +334,16 @@ nfs_update_inplace_delegation(struct nfs_delegation *delegation,
* nfs_inode_set_delegation - set up a delegation on an inode
* @inode: inode to which delegation applies
* @cred: cred to use for subsequent delegation processing
* @res: new delegation state from server
* @type: delegation type
* @stateid: delegation stateid
* @pagemod_limit: write delegation "space_limit"
*
* Returns zero on success, or a negative errno value.
*/
int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred, struct nfs_openres *res)
int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred,
fmode_t type,
const nfs4_stateid *stateid,
unsigned long pagemod_limit)
{
struct nfs_server *server = NFS_SERVER(inode);
struct nfs_client *clp = server->nfs_client;
......@@ -345,9 +355,9 @@ int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred, struct
delegation = kmalloc(sizeof(*delegation), GFP_NOFS);
if (delegation == NULL)
return -ENOMEM;
nfs4_stateid_copy(&delegation->stateid, &res->delegation);
delegation->type = res->delegation_type;
delegation->pagemod_limit = res->pagemod_limit;
nfs4_stateid_copy(&delegation->stateid, stateid);
delegation->type = type;
delegation->pagemod_limit = pagemod_limit;
delegation->change_attr = inode_peek_iversion_raw(inode);
delegation->cred = get_rpccred(cred);
delegation->inode = inode;
......@@ -392,7 +402,7 @@ int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred, struct
rcu_assign_pointer(nfsi->delegation, delegation);
delegation = NULL;
trace_nfs4_set_delegation(inode, res->delegation_type);
trace_nfs4_set_delegation(inode, type);
out:
spin_unlock(&clp->cl_lock);
......@@ -547,6 +557,22 @@ int nfs4_inode_return_delegation(struct inode *inode)
return err;
}
/**
* nfs4_inode_make_writeable
* @inode: pointer to inode
*
* Make the inode writeable by returning the delegation if necessary
*
* Returns zero on success, or a negative errno value.
*/
int nfs4_inode_make_writeable(struct inode *inode)
{
if (!nfs4_has_session(NFS_SERVER(inode)->nfs_client) ||
!nfs4_check_delegation(inode, FMODE_WRITE))
return nfs4_inode_return_delegation(inode);
return 0;
}
static void nfs_mark_return_if_closed_delegation(struct nfs_server *server,
struct nfs_delegation *delegation)
{
......
......@@ -36,8 +36,10 @@ enum {
NFS_DELEGATION_TEST_EXPIRED,
};
int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred, struct nfs_openres *res);
void nfs_inode_reclaim_delegation(struct inode *inode, struct rpc_cred *cred, struct nfs_openres *res);
int nfs_inode_set_delegation(struct inode *inode, struct rpc_cred *cred,
fmode_t type, const nfs4_stateid *stateid, unsigned long pagemod_limit);
void nfs_inode_reclaim_delegation(struct inode *inode, struct rpc_cred *cred,
fmode_t type, const nfs4_stateid *stateid, unsigned long pagemod_limit);
int nfs4_inode_return_delegation(struct inode *inode);
int nfs_async_inode_return_delegation(struct inode *inode, const nfs4_stateid *stateid);
void nfs_inode_return_delegation_noreclaim(struct inode *inode);
......@@ -70,6 +72,7 @@ int nfs4_check_delegation(struct inode *inode, fmode_t flags);
bool nfs4_delegation_flush_on_close(const struct inode *inode);
void nfs_inode_find_delegation_state_and_recover(struct inode *inode,
const nfs4_stateid *stateid);
int nfs4_inode_make_writeable(struct inode *inode);
#endif
......
......@@ -1272,7 +1272,9 @@ static void nfs_drop_nlink(struct inode *inode)
/* drop the inode if we're reasonably sure this is the last link */
if (inode->i_nlink == 1)
clear_nlink(inode);
NFS_I(inode)->cache_validity |= NFS_INO_INVALID_ATTR;
NFS_I(inode)->cache_validity |= NFS_INO_INVALID_CHANGE
| NFS_INO_INVALID_CTIME
| NFS_INO_INVALID_OTHER;
spin_unlock(&inode->i_lock);
}
......@@ -1798,12 +1800,11 @@ static int nfs_safe_remove(struct dentry *dentry)
trace_nfs_remove_enter(dir, dentry);
if (inode != NULL) {
NFS_PROTO(inode)->return_delegation(inode);
error = NFS_PROTO(dir)->remove(dir, &dentry->d_name);
error = NFS_PROTO(dir)->remove(dir, dentry);
if (error == 0)
nfs_drop_nlink(inode);
} else
error = NFS_PROTO(dir)->remove(dir, &dentry->d_name);
error = NFS_PROTO(dir)->remove(dir, dentry);
if (error == -ENOENT)
nfs_dentry_handle_enoent(dentry);
trace_nfs_remove_exit(dir, dentry, error);
......@@ -1932,8 +1933,6 @@ nfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
old_dentry, dentry);
trace_nfs_link_enter(inode, dir, dentry);
NFS_PROTO(inode)->return_delegation(inode);
d_drop(dentry);
error = NFS_PROTO(dir)->link(inode, dir, &dentry->d_name);
if (error == 0) {
......@@ -2023,10 +2022,6 @@ int nfs_rename(struct inode *old_dir, struct dentry *old_dentry,
}
}
NFS_PROTO(old_inode)->return_delegation(old_inode);
if (new_inode != NULL)
NFS_PROTO(new_inode)->return_delegation(new_inode);
task = nfs_async_rename(old_dir, new_dir, old_dentry, new_dentry, NULL);
if (IS_ERR(task)) {
error = PTR_ERR(task);
......
This diff is collapsed.
......@@ -138,8 +138,11 @@ nfs3_proc_setattr(struct dentry *dentry, struct nfs_fattr *fattr,
msg.rpc_cred = nfs_file_cred(sattr->ia_file);
nfs_fattr_init(fattr);
status = rpc_call_sync(NFS_CLIENT(inode), &msg, 0);
if (status == 0)
if (status == 0) {
if (NFS_I(inode)->cache_validity & NFS_INO_INVALID_ACL)
nfs_zap_acl_cache(inode);
nfs_setattr_update_inode(inode, sattr, fattr);
}
dprintk("NFS reply setattr: %d\n", status);
return status;
}
......@@ -383,11 +386,11 @@ nfs3_proc_create(struct inode *dir, struct dentry *dentry, struct iattr *sattr,
}
static int
nfs3_proc_remove(struct inode *dir, const struct qstr *name)
nfs3_proc_remove(struct inode *dir, struct dentry *dentry)
{
struct nfs_removeargs arg = {
.fh = NFS_FH(dir),
.name = *name,
.name = dentry->d_name,
};
struct nfs_removeres res;
struct rpc_message msg = {
......@@ -397,7 +400,7 @@ nfs3_proc_remove(struct inode *dir, const struct qstr *name)
};
int status = -ENOMEM;
dprintk("NFS call remove %s\n", name->name);
dprintk("NFS call remove %pd2\n", dentry);
res.dir_attr = nfs_alloc_fattr();
if (res.dir_attr == NULL)
goto out;
......@@ -411,7 +414,7 @@ nfs3_proc_remove(struct inode *dir, const struct qstr *name)
}
static void
nfs3_proc_unlink_setup(struct rpc_message *msg, struct inode *dir)
nfs3_proc_unlink_setup(struct rpc_message *msg, struct dentry *dentry)
{
msg->rpc_proc = &nfs3_procedures[NFS3PROC_REMOVE];
}
......@@ -433,7 +436,9 @@ nfs3_proc_unlink_done(struct rpc_task *task, struct inode *dir)
}
static void
nfs3_proc_rename_setup(struct rpc_message *msg, struct inode *dir)
nfs3_proc_rename_setup(struct rpc_message *msg,
struct dentry *old_dentry,
struct dentry *new_dentry)
{
msg->rpc_proc = &nfs3_procedures[NFS3PROC_RENAME];
}
......@@ -908,12 +913,6 @@ static int nfs3_have_delegation(struct inode *inode, fmode_t flags)
return 0;
}
static int nfs3_return_delegation(struct inode *inode)
{
nfs_wb_all(inode);
return 0;
}
static const struct inode_operations nfs3_dir_inode_operations = {
.create = nfs_create,
.lookup = nfs_lookup,
......@@ -990,7 +989,6 @@ const struct nfs_rpc_ops nfs_v3_clientops = {
.clear_acl_cache = forget_all_cached_acls,
.close_context = nfs_close_context,
.have_delegation = nfs3_have_delegation,
.return_delegation = nfs3_return_delegation,
.alloc_client = nfs_alloc_client,
.init_client = nfs_init_client,
.free_client = nfs_free_client,
......
......@@ -1997,6 +1997,7 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
struct nfs_entry old = *entry;
__be32 *p;
int error;
u64 new_cookie;
p = xdr_inline_decode(xdr, 4);
if (unlikely(p == NULL))
......@@ -2019,8 +2020,7 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
if (unlikely(error))
return error;
entry->prev_cookie = entry->cookie;
error = decode_cookie3(xdr, &entry->cookie);
error = decode_cookie3(xdr, &new_cookie);
if (unlikely(error))
return error;
......@@ -2054,6 +2054,9 @@ int nfs3_decode_dirent(struct xdr_stream *xdr, struct nfs_entry *entry,
zero_nfs_fh3(entry->fh);
}
entry->prev_cookie = entry->cookie;
entry->cookie = new_cookie;
return 0;
out_overflow:
......
This diff is collapsed.
......@@ -428,7 +428,6 @@ nfs4_insert_state_owner_locked(struct nfs4_state_owner *new)
struct rb_node **p = &server->state_owners.rb_node,
*parent = NULL;
struct nfs4_state_owner *sp;
int err;
while (*p != NULL) {
parent = *p;
......@@ -445,9 +444,6 @@ nfs4_insert_state_owner_locked(struct nfs4_state_owner *new)
return sp;
}
}
err = ida_get_new(&server->openowner_id, &new->so_seqid.owner_id);
if (err)
return ERR_PTR(err);
rb_link_node(&new->so_server_node, parent, p);
rb_insert_color(&new->so_server_node, &server->state_owners);
return new;
......@@ -460,7 +456,6 @@ nfs4_remove_state_owner_locked(struct nfs4_state_owner *sp)
if (!RB_EMPTY_NODE(&sp->so_server_node))
rb_erase(&sp->so_server_node, &server->state_owners);
ida_remove(&server->openowner_id, sp->so_seqid.owner_id);
}
static void
......@@ -495,6 +490,12 @@ nfs4_alloc_state_owner(struct nfs_server *server,
sp = kzalloc(sizeof(*sp), gfp_flags);
if (!sp)
return NULL;
sp->so_seqid.owner_id = ida_simple_get(&server->openowner_id, 0, 0,
gfp_flags);
if (sp->so_seqid.owner_id < 0) {
kfree(sp);
return NULL;
}
sp->so_server = server;
sp->so_cred = get_rpccred(cred);
spin_lock_init(&sp->so_lock);
......@@ -526,6 +527,7 @@ static void nfs4_free_state_owner(struct nfs4_state_owner *sp)
{
nfs4_destroy_seqid_counter(&sp->so_seqid);
put_rpccred(sp->so_cred);
ida_simple_remove(&sp->so_server->openowner_id, sp->so_seqid.owner_id);
kfree(sp);
}
......@@ -576,13 +578,9 @@ struct nfs4_state_owner *nfs4_get_state_owner(struct nfs_server *server,
new = nfs4_alloc_state_owner(server, cred, gfp_flags);
if (new == NULL)
goto out;
do {
if (ida_pre_get(&server->openowner_id, gfp_flags) == 0)
break;
spin_lock(&clp->cl_lock);
sp = nfs4_insert_state_owner_locked(new);
spin_unlock(&clp->cl_lock);
} while (sp == ERR_PTR(-EAGAIN));
if (sp != new)
nfs4_free_state_owner(new);
out:
......
This diff is collapsed.
......@@ -300,11 +300,11 @@ nfs_proc_mknod(struct inode *dir, struct dentry *dentry, struct iattr *sattr,
}
static int
nfs_proc_remove(struct inode *dir, const struct qstr *name)
nfs_proc_remove(struct inode *dir, struct dentry *dentry)
{
struct nfs_removeargs arg = {
.fh = NFS_FH(dir),
.name = *name,
.name = dentry->d_name,
};
struct rpc_message msg = {
.rpc_proc = &nfs_procedures[NFSPROC_REMOVE],
......@@ -312,7 +312,7 @@ nfs_proc_remove(struct inode *dir, const struct qstr *name)
};
int status;
dprintk("NFS call remove %s\n", name->name);
dprintk("NFS call remove %pd2\n",dentry);
status = rpc_call_sync(NFS_CLIENT(dir), &msg, 0);
nfs_mark_for_revalidate(dir);
......@@ -321,7 +321,7 @@ nfs_proc_remove(struct inode *dir, const struct qstr *name)
}
static void
nfs_proc_unlink_setup(struct rpc_message *msg, struct inode *dir)
nfs_proc_unlink_setup(struct rpc_message *msg, struct dentry *dentry)
{
msg->rpc_proc = &nfs_procedures[NFSPROC_REMOVE];
}
......@@ -338,7 +338,9 @@ static int nfs_proc_unlink_done(struct rpc_task *task, struct inode *dir)
}
static void
nfs_proc_rename_setup(struct rpc_message *msg, struct inode *dir)
nfs_proc_rename_setup(struct rpc_message *msg,
struct dentry *old_dentry,
struct dentry *new_dentry)
{
msg->rpc_proc = &nfs_procedures[NFSPROC_RENAME];
}
......@@ -671,12 +673,6 @@ static int nfs_have_delegation(struct inode *inode, fmode_t flags)
return 0;
}
static int nfs_return_delegation(struct inode *inode)
{
nfs_wb_all(inode);
return 0;
}
static const struct inode_operations nfs_dir_inode_operations = {
.create = nfs_create,
.lookup = nfs_lookup,
......@@ -741,7 +737,6 @@ const struct nfs_rpc_ops nfs_v2_clientops = {
.lock_check_bounds = nfs_lock_check_bounds,
.close_context = nfs_close_context,
.have_delegation = nfs_have_delegation,
.return_delegation = nfs_return_delegation,
.alloc_client = nfs_alloc_client,
.init_client = nfs_init_client,
.free_client = nfs_free_client,
......
......@@ -105,7 +105,7 @@ static void nfs_do_call_unlink(struct nfs_unlinkdata *data)
data->args.fh = NFS_FH(dir);
nfs_fattr_init(data->res.dir_attr);
NFS_PROTO(dir)->unlink_setup(&msg, dir);
NFS_PROTO(dir)->unlink_setup(&msg, data->dentry);
task_setup_data.rpc_client = NFS_CLIENT(dir);
task = rpc_run_task(&task_setup_data);
......@@ -386,7 +386,7 @@ nfs_async_rename(struct inode *old_dir, struct inode *new_dir,
nfs_sb_active(old_dir->i_sb);
NFS_PROTO(data->old_dir)->rename_setup(&msg, old_dir);
NFS_PROTO(data->old_dir)->rename_setup(&msg, old_dentry, new_dentry);
return rpc_run_task(&task_setup_data);
}
......@@ -463,9 +463,6 @@ nfs_sillyrename(struct inode *dir, struct dentry *dentry)
fileid = NFS_FILEID(d_inode(dentry));
/* Return delegation in anticipation of the rename */
NFS_PROTO(d_inode(dentry))->return_delegation(d_inode(dentry));
sdentry = NULL;
do {
int slen;
......
......@@ -231,6 +231,7 @@ static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int c
if (i_size >= end)
goto out;
i_size_write(inode, end);
NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
out:
spin_unlock(&inode->i_lock);
......@@ -1562,8 +1563,11 @@ static int nfs_writeback_done(struct rpc_task *task,
}
/* Deal with the suid/sgid bit corner case */
if (nfs_should_remove_suid(inode))
nfs_mark_for_revalidate(inode);
if (nfs_should_remove_suid(inode)) {
spin_lock(&inode->i_lock);
NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER;
spin_unlock(&inode->i_lock);
}
return 0;
}
......
......@@ -198,14 +198,24 @@ struct nfs_inode {
/*
* Cache validity bit flags
*/
#define NFS_INO_INVALID_ATTR 0x0001 /* cached attrs are invalid */
#define NFS_INO_INVALID_DATA 0x0002 /* cached data is invalid */
#define NFS_INO_INVALID_ATIME 0x0004 /* cached atime is invalid */
#define NFS_INO_INVALID_ACCESS 0x0008 /* cached access cred invalid */
#define NFS_INO_INVALID_ACL 0x0010 /* cached acls are invalid */
#define NFS_INO_REVAL_PAGECACHE 0x0020 /* must revalidate pagecache */
#define NFS_INO_REVAL_FORCED 0x0040 /* force revalidation ignoring a delegation */
#define NFS_INO_INVALID_LABEL 0x0080 /* cached label is invalid */
#define NFS_INO_INVALID_DATA BIT(1) /* cached data is invalid */
#define NFS_INO_INVALID_ATIME BIT(2) /* cached atime is invalid */
#define NFS_INO_INVALID_ACCESS BIT(3) /* cached access cred invalid */
#define NFS_INO_INVALID_ACL BIT(4) /* cached acls are invalid */
#define NFS_INO_REVAL_PAGECACHE BIT(5) /* must revalidate pagecache */
#define NFS_INO_REVAL_FORCED BIT(6) /* force revalidation ignoring a delegation */
#define NFS_INO_INVALID_LABEL BIT(7) /* cached label is invalid */
#define NFS_INO_INVALID_CHANGE BIT(8) /* cached change is invalid */
#define NFS_INO_INVALID_CTIME BIT(9) /* cached ctime is invalid */
#define NFS_INO_INVALID_MTIME BIT(10) /* cached mtime is invalid */
#define NFS_INO_INVALID_SIZE BIT(11) /* cached size is invalid */
#define NFS_INO_INVALID_OTHER BIT(12) /* other attrs are invalid */
#define NFS_INO_INVALID_ATTR (NFS_INO_INVALID_CHANGE \
| NFS_INO_INVALID_CTIME \
| NFS_INO_INVALID_MTIME \
| NFS_INO_INVALID_SIZE \
| NFS_INO_INVALID_OTHER) /* inode metadata is invalid */
/*
* Bit offsets in flags field
......@@ -292,10 +302,11 @@ static inline void nfs_mark_for_revalidate(struct inode *inode)
struct nfs_inode *nfsi = NFS_I(inode);
spin_lock(&inode->i_lock);
nfsi->cache_validity |= NFS_INO_INVALID_ATTR |
NFS_INO_REVAL_PAGECACHE |
NFS_INO_INVALID_ACCESS |
NFS_INO_INVALID_ACL;
nfsi->cache_validity |= NFS_INO_REVAL_PAGECACHE
| NFS_INO_INVALID_ACCESS
| NFS_INO_INVALID_ACL
| NFS_INO_INVALID_CHANGE
| NFS_INO_INVALID_CTIME;
if (S_ISDIR(inode->i_mode))
nfsi->cache_validity |= NFS_INO_INVALID_DATA;
spin_unlock(&inode->i_lock);
......
......@@ -1590,11 +1590,13 @@ struct nfs_rpc_ops {
unsigned int);
int (*create) (struct inode *, struct dentry *,
struct iattr *, int);
int (*remove) (struct inode *, const struct qstr *);
void (*unlink_setup) (struct rpc_message *, struct inode *dir);
int (*remove) (struct inode *, struct dentry *);
void (*unlink_setup) (struct rpc_message *, struct dentry *);
void (*unlink_rpc_prepare) (struct rpc_task *, struct nfs_unlinkdata *);
int (*unlink_done) (struct rpc_task *, struct inode *);
void (*rename_setup) (struct rpc_message *msg, struct inode *dir);
void (*rename_setup) (struct rpc_message *msg,
struct dentry *old_dentry,
struct dentry *new_dentry);
void (*rename_rpc_prepare)(struct rpc_task *task, struct nfs_renamedata *);
int (*rename_done) (struct rpc_task *task, struct inode *old_dir, struct inode *new_dir);
int (*link) (struct inode *, struct inode *, const struct qstr *);
......@@ -1633,7 +1635,6 @@ struct nfs_rpc_ops {
struct iattr *iattr,
int *);
int (*have_delegation)(struct inode *, fmode_t);
int (*return_delegation)(struct inode *);
struct nfs_client *(*alloc_client) (const struct nfs_client_initdata *);
struct nfs_client *(*init_client) (struct nfs_client *,
const struct nfs_client_initdata *);
......
......@@ -217,5 +217,12 @@ void rpc_clnt_xprt_switch_add_xprt(struct rpc_clnt *, struct rpc_xprt *);
bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt,
const struct sockaddr *sap);
void rpc_cleanup_clids(void);
static inline int rpc_reply_expected(struct rpc_task *task)
{
return (task->tk_msg.rpc_proc != NULL) &&
(task->tk_msg.rpc_proc->p_decode != NULL);
}
#endif /* __KERNEL__ */
#endif /* _LINUX_SUNRPC_CLNT_H */
......@@ -253,6 +253,12 @@ xdr_stream_remaining(const struct xdr_stream *xdr)
return xdr->nwords << 2;
}
ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr,
size_t size);
ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr,
size_t maxlen, gfp_t gfp_flags);
ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str,
size_t size);
ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str,
size_t maxlen, gfp_t gfp_flags);
/**
......@@ -312,6 +318,31 @@ xdr_stream_encode_u64(struct xdr_stream *xdr, __u64 n)
return len;
}
/**
* xdr_stream_encode_opaque_inline - Encode opaque xdr data
* @xdr: pointer to xdr_stream
* @ptr: pointer to void pointer
* @len: size of object
*
* Return values:
* On success, returns length in bytes of XDR buffer consumed
* %-EMSGSIZE on XDR buffer overflow
*/
static inline ssize_t
xdr_stream_encode_opaque_inline(struct xdr_stream *xdr, void **ptr, size_t len)
{
size_t count = sizeof(__u32) + xdr_align_size(len);
__be32 *p = xdr_reserve_space(xdr, count);
if (unlikely(!p)) {
*ptr = NULL;
return -EMSGSIZE;
}
xdr_encode_opaque(p, NULL, len);
*ptr = ++p;
return count;
}
/**
* xdr_stream_encode_opaque_fixed - Encode fixed length opaque xdr data
* @xdr: pointer to xdr_stream
......@@ -355,6 +386,31 @@ xdr_stream_encode_opaque(struct xdr_stream *xdr, const void *ptr, size_t len)
return count;
}
/**
* xdr_stream_encode_uint32_array - Encode variable length array of integers
* @xdr: pointer to xdr_stream
* @array: array of integers
* @array_size: number of elements in @array
*
* Return values:
* On success, returns length in bytes of XDR buffer consumed
* %-EMSGSIZE on XDR buffer overflow
*/
static inline ssize_t
xdr_stream_encode_uint32_array(struct xdr_stream *xdr,
const __u32 *array, size_t array_size)
{
ssize_t ret = (array_size+1) * sizeof(__u32);
__be32 *p = xdr_reserve_space(xdr, ret);
if (unlikely(!p))
return -EMSGSIZE;
*p++ = cpu_to_be32(array_size);
for (; array_size > 0; p++, array++, array_size--)
*p = cpu_to_be32p(array);
return ret;
}
/**
* xdr_stream_decode_u32 - Decode a 32-bit integer
* @xdr: pointer to xdr_stream
......@@ -432,6 +488,44 @@ xdr_stream_decode_opaque_inline(struct xdr_stream *xdr, void **ptr, size_t maxle
}
return len;
}
/**
* xdr_stream_decode_uint32_array - Decode variable length array of integers
* @xdr: pointer to xdr_stream
* @array: location to store the integer array or NULL
* @array_size: number of elements to store
*
* Return values:
* On success, returns number of elements stored in @array
* %-EBADMSG on XDR buffer overflow
* %-EMSGSIZE if the size of the array exceeds @array_size
*/
static inline ssize_t
xdr_stream_decode_uint32_array(struct xdr_stream *xdr,
__u32 *array, size_t array_size)
{
__be32 *p;
__u32 len;
ssize_t retval;
if (unlikely(xdr_stream_decode_u32(xdr, &len) < 0))
return -EBADMSG;
p = xdr_inline_decode(xdr, len * sizeof(*p));
if (unlikely(!p))
return -EBADMSG;
if (array == NULL)
return len;
if (len <= array_size) {
if (len < array_size)
memset(array+len, 0, (array_size-len)*sizeof(*array));
array_size = len;
retval = len;
} else
retval = -EMSGSIZE;
for (; array_size > 0; p++, array++, array_size--)
*array = be32_to_cpup(p);
return retval;
}
#endif /* __KERNEL__ */
#endif /* _SUNRPC_XDR_H_ */
......@@ -197,7 +197,7 @@ struct rpc_xprt {
struct list_head free; /* free slots */
unsigned int max_reqs; /* max number of slots */
unsigned int min_reqs; /* min number of slots */
atomic_t num_reqs; /* total slots */
unsigned int num_reqs; /* total slots */
unsigned long state; /* transport state */
unsigned char resvport : 1; /* use a reserved port */
atomic_t swapper; /* we're swapping over this
......@@ -373,6 +373,7 @@ void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action);
void xprt_write_space(struct rpc_xprt *xprt);
void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result);
struct rpc_rqst * xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid);
void xprt_update_rtt(struct rpc_task *task);
void xprt_complete_rqst(struct rpc_task *task, int copied);
void xprt_pin_rqst(struct rpc_rqst *req);
void xprt_unpin_rqst(struct rpc_rqst *req);
......
......@@ -50,9 +50,9 @@ DEFINE_EVENT(rpc_task_status, rpc_bind_status,
);
TRACE_EVENT(rpc_connect_status,
TP_PROTO(struct rpc_task *task, int status),
TP_PROTO(const struct rpc_task *task),
TP_ARGS(task, status),
TP_ARGS(task),
TP_STRUCT__entry(
__field(unsigned int, task_id)
......@@ -63,7 +63,7 @@ TRACE_EVENT(rpc_connect_status,
TP_fast_assign(
__entry->task_id = task->tk_pid;
__entry->client_id = task->tk_client->cl_clid;
__entry->status = status;
__entry->status = task->tk_status;
),
TP_printk("task:%u@%u status=%d",
......@@ -103,9 +103,9 @@ TRACE_EVENT(rpc_request,
DECLARE_EVENT_CLASS(rpc_task_running,
TP_PROTO(const struct rpc_clnt *clnt, const struct rpc_task *task, const void *action),
TP_PROTO(const struct rpc_task *task, const void *action),
TP_ARGS(clnt, task, action),
TP_ARGS(task, action),
TP_STRUCT__entry(
__field(unsigned int, task_id)
......@@ -117,7 +117,8 @@ DECLARE_EVENT_CLASS(rpc_task_running,
),
TP_fast_assign(
__entry->client_id = clnt ? clnt->cl_clid : -1;
__entry->client_id = task->tk_client ?
task->tk_client->cl_clid : -1;
__entry->task_id = task->tk_pid;
__entry->action = action;
__entry->runstate = task->tk_runstate;
......@@ -136,33 +137,33 @@ DECLARE_EVENT_CLASS(rpc_task_running,
DEFINE_EVENT(rpc_task_running, rpc_task_begin,
TP_PROTO(const struct rpc_clnt *clnt, const struct rpc_task *task, const void *action),
TP_PROTO(const struct rpc_task *task, const void *action),
TP_ARGS(clnt, task, action)
TP_ARGS(task, action)
);
DEFINE_EVENT(rpc_task_running, rpc_task_run_action,
TP_PROTO(const struct rpc_clnt *clnt, const struct rpc_task *task, const void *action),
TP_PROTO(const struct rpc_task *task, const void *action),
TP_ARGS(clnt, task, action)
TP_ARGS(task, action)
);
DEFINE_EVENT(rpc_task_running, rpc_task_complete,
TP_PROTO(const struct rpc_clnt *clnt, const struct rpc_task *task, const void *action),
TP_PROTO(const struct rpc_task *task, const void *action),
TP_ARGS(clnt, task, action)
TP_ARGS(task, action)
);
DECLARE_EVENT_CLASS(rpc_task_queued,
TP_PROTO(const struct rpc_clnt *clnt, const struct rpc_task *task, const struct rpc_wait_queue *q),
TP_PROTO(const struct rpc_task *task, const struct rpc_wait_queue *q),
TP_ARGS(clnt, task, q),
TP_ARGS(task, q),
TP_STRUCT__entry(
__field(unsigned int, task_id)
......@@ -175,7 +176,8 @@ DECLARE_EVENT_CLASS(rpc_task_queued,
),
TP_fast_assign(
__entry->client_id = clnt ? clnt->cl_clid : -1;
__entry->client_id = task->tk_client ?
task->tk_client->cl_clid : -1;
__entry->task_id = task->tk_pid;
__entry->timeout = task->tk_timeout;
__entry->runstate = task->tk_runstate;
......@@ -196,18 +198,63 @@ DECLARE_EVENT_CLASS(rpc_task_queued,
DEFINE_EVENT(rpc_task_queued, rpc_task_sleep,
TP_PROTO(const struct rpc_clnt *clnt, const struct rpc_task *task, const struct rpc_wait_queue *q),
TP_PROTO(const struct rpc_task *task, const struct rpc_wait_queue *q),
TP_ARGS(clnt, task, q)
TP_ARGS(task, q)
);
DEFINE_EVENT(rpc_task_queued, rpc_task_wakeup,
TP_PROTO(const struct rpc_clnt *clnt, const struct rpc_task *task, const struct rpc_wait_queue *q),
TP_PROTO(const struct rpc_task *task, const struct rpc_wait_queue *q),
TP_ARGS(task, q)
);
TRACE_EVENT(rpc_stats_latency,
TP_PROTO(
const struct rpc_task *task,
ktime_t backlog,
ktime_t rtt,
ktime_t execute
),
TP_ARGS(clnt, task, q)
TP_ARGS(task, backlog, rtt, execute),
TP_STRUCT__entry(
__field(u32, xid)
__field(int, version)
__string(progname, task->tk_client->cl_program->name)
__string(procname, rpc_proc_name(task))
__field(unsigned long, backlog)
__field(unsigned long, rtt)
__field(unsigned long, execute)
__string(addr,
task->tk_xprt->address_strings[RPC_DISPLAY_ADDR])
__string(port,
task->tk_xprt->address_strings[RPC_DISPLAY_PORT])
),
TP_fast_assign(
__entry->xid = be32_to_cpu(task->tk_rqstp->rq_xid);
__entry->version = task->tk_client->cl_vers;
__assign_str(progname, task->tk_client->cl_program->name)
__assign_str(procname, rpc_proc_name(task))
__entry->backlog = ktime_to_us(backlog);
__entry->rtt = ktime_to_us(rtt);
__entry->execute = ktime_to_us(execute);
__assign_str(addr,
task->tk_xprt->address_strings[RPC_DISPLAY_ADDR]);
__assign_str(port,
task->tk_xprt->address_strings[RPC_DISPLAY_PORT]);
),
TP_printk("peer=[%s]:%s xid=0x%08x %sv%d %s backlog=%lu rtt=%lu execute=%lu",
__get_str(addr), __get_str(port), __entry->xid,
__get_str(progname), __entry->version, __get_str(procname),
__entry->backlog, __entry->rtt, __entry->execute)
);
/*
......@@ -406,6 +453,27 @@ DEFINE_EVENT(rpc_xprt_event, xprt_complete_rqst,
TP_PROTO(struct rpc_xprt *xprt, __be32 xid, int status),
TP_ARGS(xprt, xid, status));
TRACE_EVENT(xprt_ping,
TP_PROTO(const struct rpc_xprt *xprt, int status),
TP_ARGS(xprt, status),
TP_STRUCT__entry(
__field(int, status)
__string(addr, xprt->address_strings[RPC_DISPLAY_ADDR])
__string(port, xprt->address_strings[RPC_DISPLAY_PORT])
),
TP_fast_assign(
__entry->status = status;
__assign_str(addr, xprt->address_strings[RPC_DISPLAY_ADDR]);
__assign_str(port, xprt->address_strings[RPC_DISPLAY_PORT]);
),
TP_printk("peer=[%s]:%s status=%d",
__get_str(addr), __get_str(port), __entry->status)
);
TRACE_EVENT(xs_tcp_data_ready,
TP_PROTO(struct rpc_xprt *xprt, int err, unsigned int total),
......
......@@ -1887,7 +1887,7 @@ call_connect_status(struct rpc_task *task)
dprint_status(task);
trace_rpc_connect_status(task, status);
trace_rpc_connect_status(task);
task->tk_status = 0;
switch (status) {
case -ECONNREFUSED:
......@@ -2014,6 +2014,9 @@ call_transmit_status(struct rpc_task *task)
case -EPERM:
if (RPC_IS_SOFTCONN(task)) {
xprt_end_transmit(task);
if (!task->tk_msg.rpc_proc->p_proc)
trace_xprt_ping(task->tk_xprt,
task->tk_status);
rpc_exit(task, task->tk_status);
break;
}
......@@ -2112,6 +2115,9 @@ call_status(struct rpc_task *task)
struct rpc_rqst *req = task->tk_rqstp;
int status;
if (!task->tk_msg.rpc_proc->p_proc)
trace_xprt_ping(task->tk_xprt, task->tk_status);
if (req->rq_reply_bytes_recvd > 0 && !req->rq_bytes_sent)
task->tk_status = req->rq_reply_bytes_recvd;
......
......@@ -276,7 +276,7 @@ static void rpc_set_active(struct rpc_task *task)
{
rpc_task_set_debuginfo(task);
set_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
trace_rpc_task_begin(task->tk_client, task, NULL);
trace_rpc_task_begin(task, NULL);
}
/*
......@@ -291,7 +291,7 @@ static int rpc_complete_task(struct rpc_task *task)
unsigned long flags;
int ret;
trace_rpc_task_complete(task->tk_client, task, NULL);
trace_rpc_task_complete(task, NULL);
spin_lock_irqsave(&wq->lock, flags);
clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
......@@ -358,7 +358,7 @@ static void __rpc_sleep_on_priority(struct rpc_wait_queue *q,
dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
task->tk_pid, rpc_qname(q), jiffies);
trace_rpc_task_sleep(task->tk_client, task, q);
trace_rpc_task_sleep(task, q);
__rpc_add_wait_queue(q, task, queue_priority);
......@@ -428,7 +428,7 @@ static void __rpc_do_wake_up_task_on_wq(struct workqueue_struct *wq,
return;
}
trace_rpc_task_wakeup(task->tk_client, task, queue);
trace_rpc_task_wakeup(task, queue);
__rpc_remove_wait_queue(queue, task);
......@@ -780,7 +780,7 @@ static void __rpc_execute(struct rpc_task *task)
}
if (!do_action)
break;
trace_rpc_task_run_action(task->tk_client, task, do_action);
trace_rpc_task_run_action(task, do_action);
do_action(task);
/*
......
......@@ -24,6 +24,8 @@
#include <linux/sunrpc/metrics.h>
#include <linux/rcupdate.h>
#include <trace/events/sunrpc.h>
#include "netns.h"
#define RPCDBG_FACILITY RPCDBG_MISC
......@@ -148,7 +150,7 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
struct rpc_iostats *op_metrics)
{
struct rpc_rqst *req = task->tk_rqstp;
ktime_t delta, now;
ktime_t backlog, execute, now;
if (!op_metrics || !req)
return;
......@@ -164,16 +166,20 @@ void rpc_count_iostats_metrics(const struct rpc_task *task,
op_metrics->om_bytes_sent += req->rq_xmit_bytes_sent;
op_metrics->om_bytes_recv += req->rq_reply_bytes_recvd;
backlog = 0;
if (ktime_to_ns(req->rq_xtime)) {
delta = ktime_sub(req->rq_xtime, task->tk_start);
op_metrics->om_queue = ktime_add(op_metrics->om_queue, delta);
backlog = ktime_sub(req->rq_xtime, task->tk_start);
op_metrics->om_queue = ktime_add(op_metrics->om_queue, backlog);
}
op_metrics->om_rtt = ktime_add(op_metrics->om_rtt, req->rq_rtt);
delta = ktime_sub(now, task->tk_start);
op_metrics->om_execute = ktime_add(op_metrics->om_execute, delta);
execute = ktime_sub(now, task->tk_start);
op_metrics->om_execute = ktime_add(op_metrics->om_execute, execute);
spin_unlock(&op_metrics->om_lock);
trace_rpc_stats_latency(req->rq_task, backlog, req->rq_rtt, execute);
}
EXPORT_SYMBOL_GPL(rpc_count_iostats_metrics);
......
......@@ -37,12 +37,6 @@ struct rpc_buffer {
char data[];
};
static inline int rpc_reply_expected(struct rpc_task *task)
{
return (task->tk_msg.rpc_proc != NULL) &&
(task->tk_msg.rpc_proc->p_decode != NULL);
}
static inline int sock_is_loopback(struct sock *sk)
{
struct dst_entry *dst;
......
......@@ -1518,6 +1518,88 @@ xdr_process_buf(struct xdr_buf *buf, unsigned int offset, unsigned int len,
}
EXPORT_SYMBOL_GPL(xdr_process_buf);
/**
* xdr_stream_decode_opaque - Decode variable length opaque
* @xdr: pointer to xdr_stream
* @ptr: location to store opaque data
* @size: size of storage buffer @ptr
*
* Return values:
* On success, returns size of object stored in *@ptr
* %-EBADMSG on XDR buffer overflow
* %-EMSGSIZE on overflow of storage buffer @ptr
*/
ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, size_t size)
{
ssize_t ret;
void *p;
ret = xdr_stream_decode_opaque_inline(xdr, &p, size);
if (ret <= 0)
return ret;
memcpy(ptr, p, ret);
return ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque);
/**
* xdr_stream_decode_opaque_dup - Decode and duplicate variable length opaque
* @xdr: pointer to xdr_stream
* @ptr: location to store pointer to opaque data
* @maxlen: maximum acceptable object size
* @gfp_flags: GFP mask to use
*
* Return values:
* On success, returns size of object stored in *@ptr
* %-EBADMSG on XDR buffer overflow
* %-EMSGSIZE if the size of the object would exceed @maxlen
* %-ENOMEM on memory allocation failure
*/
ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr,
size_t maxlen, gfp_t gfp_flags)
{
ssize_t ret;
void *p;
ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen);
if (ret > 0) {
*ptr = kmemdup(p, ret, gfp_flags);
if (*ptr != NULL)
return ret;
ret = -ENOMEM;
}
*ptr = NULL;
return ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque_dup);
/**
* xdr_stream_decode_string - Decode variable length string
* @xdr: pointer to xdr_stream
* @str: location to store string
* @size: size of storage buffer @str
*
* Return values:
* On success, returns length of NUL-terminated string stored in *@str
* %-EBADMSG on XDR buffer overflow
* %-EMSGSIZE on overflow of storage buffer @str
*/
ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, size_t size)
{
ssize_t ret;
void *p;
ret = xdr_stream_decode_opaque_inline(xdr, &p, size);
if (ret > 0) {
memcpy(str, p, ret);
str[ret] = '\0';
return strlen(str);
}
*str = '\0';
return ret;
}
EXPORT_SYMBOL_GPL(xdr_stream_decode_string);
/**
* xdr_stream_decode_string_dup - Decode and duplicate variable length string
* @xdr: pointer to xdr_stream
......
......@@ -826,6 +826,7 @@ static void xprt_connect_status(struct rpc_task *task)
* @xprt: transport on which the original request was transmitted
* @xid: RPC XID of incoming reply
*
* Caller holds xprt->recv_lock.
*/
struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid)
{
......@@ -834,6 +835,7 @@ struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid)
list_for_each_entry(entry, &xprt->recv, rq_list)
if (entry->rq_xid == xid) {
trace_xprt_lookup_rqst(xprt, xid, 0);
entry->rq_rtt = ktime_sub(ktime_get(), entry->rq_xtime);
return entry;
}
......@@ -889,7 +891,13 @@ __must_hold(&req->rq_xprt->recv_lock)
}
}
static void xprt_update_rtt(struct rpc_task *task)
/**
* xprt_update_rtt - Update RPC RTT statistics
* @task: RPC request that recently completed
*
* Caller holds xprt->recv_lock.
*/
void xprt_update_rtt(struct rpc_task *task)
{
struct rpc_rqst *req = task->tk_rqstp;
struct rpc_rtt *rtt = task->tk_client->cl_rtt;
......@@ -902,13 +910,14 @@ static void xprt_update_rtt(struct rpc_task *task)
rpc_set_timeo(rtt, timer, req->rq_ntrans - 1);
}
}
EXPORT_SYMBOL_GPL(xprt_update_rtt);
/**
* xprt_complete_rqst - called when reply processing is complete
* @task: RPC request that recently completed
* @copied: actual number of bytes received from the transport
*
* Caller holds transport lock.
* Caller holds xprt->recv_lock.
*/
void xprt_complete_rqst(struct rpc_task *task, int copied)
{
......@@ -920,9 +929,6 @@ void xprt_complete_rqst(struct rpc_task *task, int copied)
trace_xprt_complete_rqst(xprt, req->rq_xid, copied);
xprt->stat.recvs++;
req->rq_rtt = ktime_sub(ktime_get(), req->rq_xtime);
if (xprt->ops->timer != NULL)
xprt_update_rtt(task);
list_del_init(&req->rq_list);
req->rq_private_buf.len = copied;
......@@ -1003,7 +1009,7 @@ void xprt_transmit(struct rpc_task *task)
struct rpc_rqst *req = task->tk_rqstp;
struct rpc_xprt *xprt = req->rq_xprt;
unsigned int connect_cookie;
int status, numreqs;
int status;
dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
......@@ -1027,7 +1033,6 @@ void xprt_transmit(struct rpc_task *task)
return;
connect_cookie = xprt->connect_cookie;
req->rq_xtime = ktime_get();
status = xprt->ops->send_request(task);
trace_xprt_transmit(xprt, req->rq_xid, status);
if (status != 0) {
......@@ -1042,9 +1047,6 @@ void xprt_transmit(struct rpc_task *task)
xprt->ops->set_retrans_timeout(task);
numreqs = atomic_read(&xprt->num_reqs);
if (numreqs > xprt->stat.max_slots)
xprt->stat.max_slots = numreqs;
xprt->stat.sends++;
xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs;
xprt->stat.bklog_u += xprt->backlog.qlen;
......@@ -1106,14 +1108,15 @@ static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt)
{
struct rpc_rqst *req = ERR_PTR(-EAGAIN);
if (!atomic_add_unless(&xprt->num_reqs, 1, xprt->max_reqs))
if (xprt->num_reqs >= xprt->max_reqs)
goto out;
++xprt->num_reqs;
spin_unlock(&xprt->reserve_lock);
req = kzalloc(sizeof(struct rpc_rqst), GFP_NOFS);
spin_lock(&xprt->reserve_lock);
if (req != NULL)
goto out;
atomic_dec(&xprt->num_reqs);
--xprt->num_reqs;
req = ERR_PTR(-ENOMEM);
out:
return req;
......@@ -1121,7 +1124,8 @@ static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt)
static bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
{
if (atomic_add_unless(&xprt->num_reqs, -1, xprt->min_reqs)) {
if (xprt->num_reqs > xprt->min_reqs) {
--xprt->num_reqs;
kfree(req);
return true;
}
......@@ -1157,6 +1161,8 @@ void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
spin_unlock(&xprt->reserve_lock);
return;
out_init_req:
xprt->stat.max_slots = max_t(unsigned int, xprt->stat.max_slots,
xprt->num_reqs);
task->tk_status = 0;
task->tk_rqstp = req;
xprt_request_init(task, xprt);
......@@ -1224,7 +1230,7 @@ struct rpc_xprt *xprt_alloc(struct net *net, size_t size,
else
xprt->max_reqs = num_prealloc;
xprt->min_reqs = num_prealloc;
atomic_set(&xprt->num_reqs, num_prealloc);
xprt->num_reqs = num_prealloc;
return xprt;
......
......@@ -44,13 +44,6 @@ static int rpcrdma_bc_setup_rqst(struct rpcrdma_xprt *r_xprt,
if (IS_ERR(req))
return PTR_ERR(req);
rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE,
DMA_TO_DEVICE, GFP_KERNEL);
if (IS_ERR(rb))
goto out_fail;
req->rl_rdmabuf = rb;
xdr_buf_init(&req->rl_hdrbuf, rb->rg_base, rdmab_length(rb));
size = r_xprt->rx_data.inline_rsize;
rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, GFP_KERNEL);
if (IS_ERR(rb))
......
......@@ -191,7 +191,7 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
mr = rpcrdma_mr_get(r_xprt);
if (!mr)
return ERR_PTR(-ENOBUFS);
return ERR_PTR(-EAGAIN);
pageoff = offset_in_page(seg1->mr_offset);
seg1->mr_offset -= pageoff; /* start of page */
......@@ -251,6 +251,16 @@ fmr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
return ERR_PTR(-EIO);
}
/* Post Send WR containing the RPC Call message.
*/
static int
fmr_op_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req)
{
struct ib_send_wr *bad_wr;
return ib_post_send(ia->ri_id->qp, &req->rl_sendctx->sc_wr, &bad_wr);
}
/* Invalidate all memory regions that were registered for "req".
*
* Sleeps until it is safe for the host CPU to access the
......@@ -305,6 +315,7 @@ fmr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs)
const struct rpcrdma_memreg_ops rpcrdma_fmr_memreg_ops = {
.ro_map = fmr_op_map,
.ro_send = fmr_op_send,
.ro_unmap_sync = fmr_op_unmap_sync,
.ro_recover_mr = fmr_op_recover_mr,
.ro_open = fmr_op_open,
......
......@@ -357,8 +357,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
struct rpcrdma_mr *mr;
struct ib_mr *ibmr;
struct ib_reg_wr *reg_wr;
struct ib_send_wr *bad_wr;
int rc, i, n;
int i, n;
u8 key;
mr = NULL;
......@@ -367,7 +366,7 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
rpcrdma_mr_defer_recovery(mr);
mr = rpcrdma_mr_get(r_xprt);
if (!mr)
return ERR_PTR(-ENOBUFS);
return ERR_PTR(-EAGAIN);
} while (mr->frwr.fr_state != FRWR_IS_INVALID);
frwr = &mr->frwr;
frwr->fr_state = FRWR_IS_VALID;
......@@ -407,22 +406,12 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
ib_update_fast_reg_key(ibmr, ++key);
reg_wr = &frwr->fr_regwr;
reg_wr->wr.next = NULL;
reg_wr->wr.opcode = IB_WR_REG_MR;
frwr->fr_cqe.done = frwr_wc_fastreg;
reg_wr->wr.wr_cqe = &frwr->fr_cqe;
reg_wr->wr.num_sge = 0;
reg_wr->wr.send_flags = 0;
reg_wr->mr = ibmr;
reg_wr->key = ibmr->rkey;
reg_wr->access = writing ?
IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE :
IB_ACCESS_REMOTE_READ;
rc = ib_post_send(ia->ri_id->qp, &reg_wr->wr, &bad_wr);
if (rc)
goto out_senderr;
mr->mr_handle = ibmr->rkey;
mr->mr_length = ibmr->length;
mr->mr_offset = ibmr->iova;
......@@ -442,11 +431,40 @@ frwr_op_map(struct rpcrdma_xprt *r_xprt, struct rpcrdma_mr_seg *seg,
frwr->fr_mr, n, mr->mr_nents);
rpcrdma_mr_defer_recovery(mr);
return ERR_PTR(-EIO);
}
out_senderr:
pr_err("rpcrdma: FRWR registration ib_post_send returned %i\n", rc);
rpcrdma_mr_defer_recovery(mr);
return ERR_PTR(-ENOTCONN);
/* Post Send WR containing the RPC Call message.
*
* For FRMR, chain any FastReg WRs to the Send WR. Only a
* single ib_post_send call is needed to register memory
* and then post the Send WR.
*/
static int
frwr_op_send(struct rpcrdma_ia *ia, struct rpcrdma_req *req)
{
struct ib_send_wr *post_wr, *bad_wr;
struct rpcrdma_mr *mr;
post_wr = &req->rl_sendctx->sc_wr;
list_for_each_entry(mr, &req->rl_registered, mr_list) {
struct rpcrdma_frwr *frwr;
frwr = &mr->frwr;
frwr->fr_cqe.done = frwr_wc_fastreg;
frwr->fr_regwr.wr.next = post_wr;
frwr->fr_regwr.wr.wr_cqe = &frwr->fr_cqe;
frwr->fr_regwr.wr.num_sge = 0;
frwr->fr_regwr.wr.opcode = IB_WR_REG_MR;
frwr->fr_regwr.wr.send_flags = 0;
post_wr = &frwr->fr_regwr.wr;
}
/* If ib_post_send fails, the next ->send_request for
* @req will queue these MWs for recovery.
*/
return ib_post_send(ia->ri_id->qp, post_wr, &bad_wr);
}
/* Handle a remotely invalidated mr on the @mrs list
......@@ -561,6 +579,7 @@ frwr_op_unmap_sync(struct rpcrdma_xprt *r_xprt, struct list_head *mrs)
const struct rpcrdma_memreg_ops rpcrdma_frwr_memreg_ops = {
.ro_map = frwr_op_map,
.ro_send = frwr_op_send,
.ro_reminv = frwr_op_reminv,
.ro_unmap_sync = frwr_op_unmap_sync,
.ro_recover_mr = frwr_op_recover_mr,
......
......@@ -365,7 +365,7 @@ rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,
false, &mr);
if (IS_ERR(seg))
return PTR_ERR(seg);
goto out_maperr;
rpcrdma_mr_push(mr, &req->rl_registered);
if (encode_read_segment(xdr, mr, pos) < 0)
......@@ -377,6 +377,11 @@ rpcrdma_encode_read_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
} while (nsegs);
return 0;
out_maperr:
if (PTR_ERR(seg) == -EAGAIN)
xprt_wait_for_buffer_space(rqst->rq_task, NULL);
return PTR_ERR(seg);
}
/* Register and XDR encode the Write list. Supports encoding a list
......@@ -423,7 +428,7 @@ rpcrdma_encode_write_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,
true, &mr);
if (IS_ERR(seg))
return PTR_ERR(seg);
goto out_maperr;
rpcrdma_mr_push(mr, &req->rl_registered);
if (encode_rdma_segment(xdr, mr) < 0)
......@@ -440,6 +445,11 @@ rpcrdma_encode_write_list(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
*segcount = cpu_to_be32(nchunks);
return 0;
out_maperr:
if (PTR_ERR(seg) == -EAGAIN)
xprt_wait_for_buffer_space(rqst->rq_task, NULL);
return PTR_ERR(seg);
}
/* Register and XDR encode the Reply chunk. Supports encoding an array
......@@ -481,7 +491,7 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
seg = r_xprt->rx_ia.ri_ops->ro_map(r_xprt, seg, nsegs,
true, &mr);
if (IS_ERR(seg))
return PTR_ERR(seg);
goto out_maperr;
rpcrdma_mr_push(mr, &req->rl_registered);
if (encode_rdma_segment(xdr, mr) < 0)
......@@ -498,6 +508,11 @@ rpcrdma_encode_reply_chunk(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
*segcount = cpu_to_be32(nchunks);
return 0;
out_maperr:
if (PTR_ERR(seg) == -EAGAIN)
xprt_wait_for_buffer_space(rqst->rq_task, NULL);
return PTR_ERR(seg);
}
/**
......@@ -724,8 +739,8 @@ rpcrdma_prepare_send_sges(struct rpcrdma_xprt *r_xprt,
* Returns:
* %0 if the RPC was sent successfully,
* %-ENOTCONN if the connection was lost,
* %-EAGAIN if not enough pages are available for on-demand reply buffer,
* %-ENOBUFS if no MRs are available to register chunks,
* %-EAGAIN if the caller should call again with the same arguments,
* %-ENOBUFS if the caller should call again after a delay,
* %-EMSGSIZE if the transport header is too small,
* %-EIO if a permanent problem occurred while marshaling.
*/
......@@ -868,10 +883,7 @@ rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst)
return 0;
out_err:
if (ret != -ENOBUFS) {
pr_err("rpcrdma: header marshaling failed (%d)\n", ret);
r_xprt->rx_stats.failed_marshal_count++;
}
return ret;
}
......@@ -1366,7 +1378,7 @@ void rpcrdma_reply_handler(struct rpcrdma_rep *rep)
trace_xprtrdma_reply(rqst->rq_task, rep, req, credits);
queue_work_on(req->rl_cpu, rpcrdma_receive_wq, &rep->rr_work);
queue_work(rpcrdma_receive_wq, &rep->rr_work);
return;
out_badstatus:
......
......@@ -52,7 +52,6 @@
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/sunrpc/addr.h>
#include <linux/smp.h>
#include "xprt_rdma.h"
......@@ -237,8 +236,6 @@ rpcrdma_connect_worker(struct work_struct *work)
struct rpc_xprt *xprt = &r_xprt->rx_xprt;
spin_lock_bh(&xprt->transport_lock);
if (++xprt->connect_cookie == 0) /* maintain a reserved value */
++xprt->connect_cookie;
if (ep->rep_connected > 0) {
if (!xprt_test_and_set_connected(xprt))
xprt_wake_pending_tasks(xprt, 0);
......@@ -540,29 +537,6 @@ xprt_rdma_connect(struct rpc_xprt *xprt, struct rpc_task *task)
}
}
/* Allocate a fixed-size buffer in which to construct and send the
* RPC-over-RDMA header for this request.
*/
static bool
rpcrdma_get_rdmabuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
gfp_t flags)
{
size_t size = RPCRDMA_HDRBUF_SIZE;
struct rpcrdma_regbuf *rb;
if (req->rl_rdmabuf)
return true;
rb = rpcrdma_alloc_regbuf(size, DMA_TO_DEVICE, flags);
if (IS_ERR(rb))
return false;
r_xprt->rx_stats.hardway_register_count += size;
req->rl_rdmabuf = rb;
xdr_buf_init(&req->rl_hdrbuf, rb->rg_base, rdmab_length(rb));
return true;
}
static bool
rpcrdma_get_sendbuf(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req,
size_t size, gfp_t flags)
......@@ -644,15 +618,11 @@ xprt_rdma_allocate(struct rpc_task *task)
if (RPC_IS_SWAPPER(task))
flags = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN;
if (!rpcrdma_get_rdmabuf(r_xprt, req, flags))
goto out_fail;
if (!rpcrdma_get_sendbuf(r_xprt, req, rqst->rq_callsize, flags))
goto out_fail;
if (!rpcrdma_get_recvbuf(r_xprt, req, rqst->rq_rcvsize, flags))
goto out_fail;
req->rl_cpu = smp_processor_id();
req->rl_connect_cookie = 0; /* our reserved value */
rpcrdma_set_xprtdata(rqst, req);
rqst->rq_buffer = req->rl_sendbuf->rg_base;
rqst->rq_rbuffer = req->rl_recvbuf->rg_base;
......@@ -694,7 +664,8 @@ xprt_rdma_free(struct rpc_task *task)
* Returns:
* %0 if the RPC message has been sent
* %-ENOTCONN if the caller should reconnect and call again
* %-ENOBUFS if the caller should call again later
* %-EAGAIN if the caller should call again
* %-ENOBUFS if the caller should call again after a delay
* %-EIO if a permanent error occurred and the request was not
* sent. Do not try to send this message again.
*/
......@@ -723,9 +694,9 @@ xprt_rdma_send_request(struct rpc_task *task)
rpcrdma_recv_buffer_get(req);
/* Must suppress retransmit to maintain credits */
if (req->rl_connect_cookie == xprt->connect_cookie)
if (rqst->rq_connect_cookie == xprt->connect_cookie)
goto drop_connection;
req->rl_connect_cookie = xprt->connect_cookie;
rqst->rq_xtime = ktime_get();
__set_bit(RPCRDMA_REQ_F_PENDING, &req->rl_flags);
if (rpcrdma_ep_post(&r_xprt->rx_ia, &r_xprt->rx_ep, req))
......@@ -733,6 +704,12 @@ xprt_rdma_send_request(struct rpc_task *task)
rqst->rq_xmit_bytes_sent += rqst->rq_snd_buf.len;
rqst->rq_bytes_sent = 0;
/* An RPC with no reply will throw off credit accounting,
* so drop the connection to reset the credit grant.
*/
if (!rpc_reply_expected(task))
goto drop_connection;
return 0;
failed_marshal:
......
......@@ -250,11 +250,11 @@ rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
wait_for_completion(&ia->ri_remove_done);
ia->ri_id = NULL;
ia->ri_pd = NULL;
ia->ri_device = NULL;
/* Return 1 to ensure the core destroys the id. */
return 1;
case RDMA_CM_EVENT_ESTABLISHED:
++xprt->rx_xprt.connect_cookie;
connstate = 1;
rpcrdma_update_connect_private(xprt, &event->param.conn);
goto connected;
......@@ -273,6 +273,7 @@ rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
connstate = -EAGAIN;
goto connected;
case RDMA_CM_EVENT_DISCONNECTED:
++xprt->rx_xprt.connect_cookie;
connstate = -ECONNABORTED;
connected:
xprt->rx_buf.rb_credits = 1;
......@@ -445,7 +446,9 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia)
ia->ri_id->qp = NULL;
}
ib_free_cq(ep->rep_attr.recv_cq);
ep->rep_attr.recv_cq = NULL;
ib_free_cq(ep->rep_attr.send_cq);
ep->rep_attr.send_cq = NULL;
/* The ULP is responsible for ensuring all DMA
* mappings and MRs are gone.
......@@ -458,6 +461,8 @@ rpcrdma_ia_remove(struct rpcrdma_ia *ia)
rpcrdma_dma_unmap_regbuf(req->rl_recvbuf);
}
rpcrdma_mrs_destroy(buf);
ib_dealloc_pd(ia->ri_pd);
ia->ri_pd = NULL;
/* Allow waiters to continue */
complete(&ia->ri_remove_done);
......@@ -589,11 +594,8 @@ rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
/* Client offers RDMA Read but does not initiate */
ep->rep_remote_cma.initiator_depth = 0;
if (ia->ri_device->attrs.max_qp_rd_atom > 32) /* arbitrary but <= 255 */
ep->rep_remote_cma.responder_resources = 32;
else
ep->rep_remote_cma.responder_resources =
ia->ri_device->attrs.max_qp_rd_atom;
min_t(int, U8_MAX, ia->ri_device->attrs.max_qp_rd_atom);
/* Limit transport retries so client can detect server
* GID changes quickly. RPC layer handles re-establishing
......@@ -628,13 +630,15 @@ rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
{
cancel_delayed_work_sync(&ep->rep_connect_worker);
if (ia->ri_id->qp) {
if (ia->ri_id && ia->ri_id->qp) {
rpcrdma_ep_disconnect(ep, ia);
rdma_destroy_qp(ia->ri_id);
ia->ri_id->qp = NULL;
}
if (ep->rep_attr.recv_cq)
ib_free_cq(ep->rep_attr.recv_cq);
if (ep->rep_attr.send_cq)
ib_free_cq(ep->rep_attr.send_cq);
}
......@@ -1024,7 +1028,7 @@ rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt)
LIST_HEAD(free);
LIST_HEAD(all);
for (count = 0; count < 32; count++) {
for (count = 0; count < 3; count++) {
struct rpcrdma_mr *mr;
int rc;
......@@ -1049,8 +1053,9 @@ rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt)
list_splice(&all, &buf->rb_all);
r_xprt->rx_stats.mrs_allocated += count;
spin_unlock(&buf->rb_mrlock);
trace_xprtrdma_createmrs(r_xprt, count);
xprt_write_space(&r_xprt->rx_xprt);
}
static void
......@@ -1068,17 +1073,27 @@ struct rpcrdma_req *
rpcrdma_create_req(struct rpcrdma_xprt *r_xprt)
{
struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;
struct rpcrdma_regbuf *rb;
struct rpcrdma_req *req;
req = kzalloc(sizeof(*req), GFP_KERNEL);
if (req == NULL)
return ERR_PTR(-ENOMEM);
rb = rpcrdma_alloc_regbuf(RPCRDMA_HDRBUF_SIZE,
DMA_TO_DEVICE, GFP_KERNEL);
if (IS_ERR(rb)) {
kfree(req);
return ERR_PTR(-ENOMEM);
}
req->rl_rdmabuf = rb;
xdr_buf_init(&req->rl_hdrbuf, rb->rg_base, rdmab_length(rb));
req->rl_buffer = buffer;
INIT_LIST_HEAD(&req->rl_registered);
spin_lock(&buffer->rb_reqslock);
list_add(&req->rl_all, &buffer->rb_allreqs);
spin_unlock(&buffer->rb_reqslock);
req->rl_buffer = &r_xprt->rx_buf;
INIT_LIST_HEAD(&req->rl_registered);
return req;
}
......@@ -1535,7 +1550,6 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia,
struct rpcrdma_req *req)
{
struct ib_send_wr *send_wr = &req->rl_sendctx->sc_wr;
struct ib_send_wr *send_wr_fail;
int rc;
if (req->rl_reply) {
......@@ -1554,7 +1568,7 @@ rpcrdma_ep_post(struct rpcrdma_ia *ia,
--ep->rep_send_count;
}
rc = ib_post_send(ia->ri_id->qp, send_wr, &send_wr_fail);
rc = ia->ri_ops->ro_send(ia, req);
trace_xprtrdma_post_send(req, rc);
if (rc)
return -ENOTCONN;
......
......@@ -334,8 +334,6 @@ enum {
struct rpcrdma_buffer;
struct rpcrdma_req {
struct list_head rl_list;
int rl_cpu;
unsigned int rl_connect_cookie;
struct rpcrdma_buffer *rl_buffer;
struct rpcrdma_rep *rl_reply;
struct xdr_stream rl_stream;
......@@ -474,6 +472,8 @@ struct rpcrdma_memreg_ops {
(*ro_map)(struct rpcrdma_xprt *,
struct rpcrdma_mr_seg *, int, bool,
struct rpcrdma_mr **);
int (*ro_send)(struct rpcrdma_ia *ia,
struct rpcrdma_req *req);
void (*ro_reminv)(struct rpcrdma_rep *rep,
struct list_head *mrs);
void (*ro_unmap_sync)(struct rpcrdma_xprt *,
......
......@@ -527,6 +527,7 @@ static int xs_local_send_request(struct rpc_task *task)
xs_pktdump("packet data:",
req->rq_svec->iov_base, req->rq_svec->iov_len);
req->rq_xtime = ktime_get();
status = xs_sendpages(transport->sock, NULL, 0, xdr, req->rq_bytes_sent,
true, &sent);
dprintk("RPC: %s(%u) = %d\n",
......@@ -589,6 +590,7 @@ static int xs_udp_send_request(struct rpc_task *task)
if (!xprt_bound(xprt))
return -ENOTCONN;
req->rq_xtime = ktime_get();
status = xs_sendpages(transport->sock, xs_addr(xprt), xprt->addrlen,
xdr, req->rq_bytes_sent, true, &sent);
......@@ -678,6 +680,7 @@ static int xs_tcp_send_request(struct rpc_task *task)
/* Continue transmitting the packet/record. We must be careful
* to cope with writespace callbacks arriving _after_ we have
* called sendmsg(). */
req->rq_xtime = ktime_get();
while (1) {
sent = 0;
status = xs_sendpages(transport->sock, NULL, 0, xdr,
......@@ -1060,6 +1063,7 @@ static void xs_udp_data_read_skb(struct rpc_xprt *xprt,
if (!rovr)
goto out_unlock;
xprt_pin_rqst(rovr);
xprt_update_rtt(rovr->rq_task);
spin_unlock(&xprt->recv_lock);
task = rovr->rq_task;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment