NFSv4 state model update

  - Hierarchy of state attached to nfs4_client in order to
    simplify state recovery.
    state_owners hang off nfs4_client, whereas state hangs
    off both open_owners and the nfs_inode.
    
  - Model tries to minimize the number of open_owners on
    the server by recycling unused open_owners into a pool.
    
  - NFSv4 state attached to file->private_data. Previously
    this was used by credentials (and still is for NFSv2/v3)
    Abstract out setup/release of struct file and nfs_page
    structure initialization in order to cope with these
    conflicting uses of private_data.

fs/nfs/dir.c

@@ -785,6 +785,9 @@ static int nfs_instantiate(struct dentry *dentry, struct nfs_fh *fhandle,
 	struct inode *inode;
 	int error = -EACCES;
 
+	/* We may have been initialized further down */
+	if (dentry->d_inode)
+		return 0;
 	if (fhandle->size == 0 || !(fattr->valid & NFS_ATTR_FATTR)) {
 		struct inode *dir = dentry->d_parent->d_inode;
 		error = NFS_PROTO(dir)->lookup(dir, &dentry->d_name, fhandle, fattr);
@@ -813,8 +816,7 @@ static int nfs_create(struct inode *dir, struct dentry *dentry, int mode,
 		struct nameidata *nd)
 {
 	struct iattr attr;
-	struct nfs_fattr fattr;
-	struct nfs_fh fhandle;
+	struct inode *inode;
 	int error;
 	int open_flags = 0;
 
@@ -835,12 +837,15 @@ static int nfs_create(struct inode *dir, struct dentry *dentry, int mode,
 	 */
 	lock_kernel();
 	nfs_zap_caches(dir);
-	error = NFS_PROTO(dir)->create(dir, &dentry->d_name,
-					 &attr, open_flags, &fhandle, &fattr);
-	if (!error)
-		error = nfs_instantiate(dentry, &fhandle, &fattr);
-	else
+	inode = NFS_PROTO(dir)->create(dir, &dentry->d_name, &attr, open_flags);
+	if (!IS_ERR(inode)) {
+		d_instantiate(dentry, inode);
+		nfs_renew_times(dentry);
+		error = 0;
+	} else {
+		error = PTR_ERR(inode);
 		d_drop(dentry);
+	}
 	unlock_kernel();
 	return error;
 }

fs/nfs/file.c

@@ -35,6 +35,7 @@
 #define NFSDBG_FACILITY		NFSDBG_FILE
 
 static int nfs_file_open(struct inode *, struct file *);
+static int nfs_file_release(struct inode *, struct file *);
 static int  nfs_file_mmap(struct file *, struct vm_area_struct *);
 static ssize_t nfs_file_sendfile(struct file *, loff_t *, size_t, read_actor_t, void *);
 static ssize_t nfs_file_read(struct kiocb *, char *, size_t, loff_t);
@@ -51,7 +52,7 @@ struct file_operations nfs_file_operations = {
 	.mmap		= nfs_file_mmap,
 	.open		= nfs_file_open,
 	.flush		= nfs_file_flush,
-	.release	= nfs_release,
+	.release	= nfs_file_release,
 	.fsync		= nfs_fsync,
 	.lock		= nfs_lock,
 	.sendfile	= nfs_file_sendfile,
@@ -82,13 +83,16 @@ nfs_file_open(struct inode *inode, struct file *filp)
 	/* Do NFSv4 open() call */
 	if ((open = server->rpc_ops->file_open) != NULL)
 		res = open(inode, filp);
-	/* Call generic open code in order to cache credentials */
-	if (!res)
-		res = nfs_open(inode, filp);
 	unlock_kernel();
 	return res;
 }
 
+static int
+nfs_file_release(struct inode *inode, struct file *filp)
+{
+	return NFS_PROTO(inode)->file_release(inode, filp);
+}
+
 /*
  * Flush all dirty pages, and check for write errors.
  *

fs/nfs/inode.c

@@ -151,10 +151,6 @@ nfs_clear_inode(struct inode *inode)
 	cred = nfsi->cache_access.cred;
 	if (cred)
 		put_rpccred(cred);
-	/* Clean up the V4 state */
-	nfs4_put_state_owner(inode, nfsi->wo_owner);
-	nfs4_put_state_owner(inode, nfsi->ro_owner);
-	nfs4_put_state_owner(inode, nfsi->rw_owner);
 }
 
 void
@@ -891,7 +887,7 @@ int nfs_getattr(struct vfsmount *mnt, struct dentry *dentry, struct kstat *stat)
  * Ensure that mmap has a recent RPC credential for use when writing out
  * shared pages
  */
-static inline void
+void
 nfs_set_mmcred(struct inode *inode, struct rpc_cred *cred)
 {
 	struct rpc_cred **p = &NFS_I(inode)->mm_cred,
@@ -1573,9 +1569,7 @@ static struct file_system_type nfs4_fs_type = {
 
 #define nfs4_zero_state(nfsi) \
 	do { \
-		(nfsi)->wo_owner = NULL; \
-		(nfsi)->ro_owner = NULL; \
-		(nfsi)->rw_owner = NULL; \
+		INIT_LIST_HEAD(&(nfsi)->open_states); \
 	} while(0)
 #define register_nfs4fs() register_filesystem(&nfs4_fs_type)
 #define unregister_nfs4fs() unregister_filesystem(&nfs4_fs_type)

fs/nfs/nfs3proc.c

@@ -81,6 +81,18 @@ nfs3_write_refresh_inode(struct inode *inode, struct nfs_fattr *fattr)
 	}
 }
 
+static struct rpc_cred *
+nfs_cred(struct inode *inode, struct file *filp)
+{
+	struct rpc_cred *cred = NULL;
+
+	if (filp)
+		cred = (struct rpc_cred *)filp->private_data;
+	if (!cred)
+		cred = NFS_I(inode)->mm_cred;
+	return cred;
+}
+
 /*
  * Bare-bones access to getattr: this is for nfs_read_super.
  */
@@ -225,7 +237,7 @@ nfs3_proc_readlink(struct inode *inode, struct page *page)
 }
 
 static int
-nfs3_proc_read(struct nfs_read_data *rdata)
+nfs3_proc_read(struct nfs_read_data *rdata, struct file *filp)
 {
 	int			flags = rdata->flags;
 	struct inode *		inode = rdata->inode;
@@ -234,13 +246,13 @@ nfs3_proc_read(struct nfs_read_data *rdata)
 		.rpc_proc	= &nfs3_procedures[NFS3PROC_READ],
 		.rpc_argp	= &rdata->args,
 		.rpc_resp	= &rdata->res,
-		.rpc_cred	= rdata->cred,
 	};
 	int			status;
 
 	dprintk("NFS call  read %d @ %Ld\n", rdata->args.count,
 			(long long) rdata->args.offset);
 	fattr->valid = 0;
+	msg.rpc_cred = nfs_cred(inode, filp);
 	status = rpc_call_sync(NFS_CLIENT(inode), &msg, flags);
 	if (status >= 0)
 		nfs_refresh_inode(inode, fattr);
@@ -249,7 +261,7 @@ nfs3_proc_read(struct nfs_read_data *rdata)
 }
 
 static int
-nfs3_proc_write(struct nfs_write_data *wdata)
+nfs3_proc_write(struct nfs_write_data *wdata, struct file *filp)
 {
 	int			rpcflags = wdata->flags;
 	struct inode *		inode = wdata->inode;
@@ -258,13 +270,13 @@ nfs3_proc_write(struct nfs_write_data *wdata)
 		.rpc_proc	= &nfs3_procedures[NFS3PROC_WRITE],
 		.rpc_argp	= &wdata->args,
 		.rpc_resp	= &wdata->res,
-		.rpc_cred	= wdata->cred,
 	};
 	int			status;
 
 	dprintk("NFS call  write %d @ %Ld\n", wdata->args.count,
 			(long long) wdata->args.offset);
 	fattr->valid = 0;
+	msg.rpc_cred = nfs_cred(inode, filp);
 	status = rpc_call_sync(NFS_CLIENT(inode), &msg, rpcflags);
 	if (status >= 0)
 		nfs3_write_refresh_inode(inode, fattr);
@@ -276,10 +288,12 @@ nfs3_proc_write(struct nfs_write_data *wdata)
  * Create a regular file.
  * For now, we don't implement O_EXCL.
  */
-static int
+static struct inode *
 nfs3_proc_create(struct inode *dir, struct qstr *name, struct iattr *sattr,
-		 int flags, struct nfs_fh *fhandle, struct nfs_fattr *fattr)
+		 int flags)
 {
+	struct nfs_fh		fhandle;
+	struct nfs_fattr	fattr;
 	struct nfs_fattr	dir_attr;
 	struct nfs3_createargs	arg = {
 		.fh		= NFS_FH(dir),
@@ -289,8 +303,8 @@ nfs3_proc_create(struct inode *dir, struct qstr *name, struct iattr *sattr,
 	};
 	struct nfs3_diropres	res = {
 		.dir_attr	= &dir_attr,
-		.fh		= fhandle,
-		.fattr		= fattr
+		.fh		= &fhandle,
+		.fattr		= &fattr
 	};
 	int			status;
 
@@ -304,7 +318,7 @@ nfs3_proc_create(struct inode *dir, struct qstr *name, struct iattr *sattr,
 
 again:
 	dir_attr.valid = 0;
-	fattr->valid = 0;
+	fattr.valid = 0;
 	status = rpc_call(NFS_CLIENT(dir), NFS3PROC_CREATE, &arg, &res, 0);
 	nfs_refresh_inode(dir, &dir_attr);
 
@@ -329,11 +343,19 @@ nfs3_proc_create(struct inode *dir, struct qstr *name, struct iattr *sattr,
 exit:
 	dprintk("NFS reply create: %d\n", status);
 
+	if (status != 0)
+		goto out;
+	if (fhandle.size == 0 || !(fattr.valid & NFS_ATTR_FATTR)) {
+		status = nfs3_proc_lookup(dir, name, &fhandle, &fattr);
+		if (status != 0)
+			goto out;
+	}
+
 	/* When we created the file with exclusive semantics, make
 	 * sure we set the attributes afterwards. */
-	if (status == 0 && arg.createmode == NFS3_CREATE_EXCLUSIVE) {
+	if (arg.createmode == NFS3_CREATE_EXCLUSIVE) {
 		struct nfs3_sattrargs	arg = {
-			.fh		= fhandle,
+			.fh		= &fhandle,
 			.sattr		= sattr,
 		};
 		dprintk("NFS call  setattr (post-create)\n");
@@ -341,13 +363,20 @@ nfs3_proc_create(struct inode *dir, struct qstr *name, struct iattr *sattr,
 		/* Note: we could use a guarded setattr here, but I'm
 		 * not sure this buys us anything (and I'd have
 		 * to revamp the NFSv3 XDR code) */
-		fattr->valid = 0;
+		fattr.valid = 0;
 		status = rpc_call(NFS_CLIENT(dir), NFS3PROC_SETATTR,
-						&arg, fattr, 0);
+						&arg, &fattr, 0);
 		dprintk("NFS reply setattr (post-create): %d\n", status);
 	}
-
-	return status;
+	if (status == 0) {
+		struct inode *inode;
+		inode = nfs_fhget(dir->i_sb, &fhandle, &fattr);
+		if (inode)
+			return inode;
+		status = -ENOMEM;
+	}
+out:
+	return ERR_PTR(status);
 }
 
 static int
@@ -823,6 +852,27 @@ nfs3_proc_commit_setup(struct nfs_write_data *data, u64 start, u32 len, int how)
 	rpc_call_setup(&data->task, &msg, 0);
 }
 
+/*
+ * Set up the nfspage struct with the right credentials
+ */
+void
+nfs3_request_init(struct nfs_page *req, struct file *filp)
+{
+	req->wb_cred = get_rpccred(nfs_cred(req->wb_inode, filp));
+}
+
+static int
+nfs3_request_compatible(struct nfs_page *req, struct file *filp, struct page *page)
+{
+	if (req->wb_file != filp)
+		return 0;
+	if (req->wb_page != page)
+		return 0;
+	if (req->wb_cred != nfs_file_cred(filp))
+		return 0;
+	return 1;
+}
+
 struct nfs_rpc_ops	nfs_v3_clientops = {
 	.version	= 3,			/* protocol version */
 	.getroot	= nfs3_proc_get_root,
@@ -851,4 +901,8 @@ struct nfs_rpc_ops	nfs_v3_clientops = {
 	.read_setup	= nfs3_proc_read_setup,
 	.write_setup	= nfs3_proc_write_setup,
 	.commit_setup	= nfs3_proc_commit_setup,
+	.file_open	= nfs_open,
+	.file_release	= nfs_release,
+	.request_init	= nfs3_request_init,
+	.request_compatible = nfs3_request_compatible,
 };

fs/nfs/nfs4state.c

@@ -42,8 +42,9 @@
 #include <linux/slab.h>
 #include <linux/nfs_fs.h>
 
-/* This protects most of the client-side state. */
-static spinlock_t               state_spinlock = SPIN_LOCK_UNLOCKED;
+#define OPENOWNER_POOL_SIZE	8
+
+static spinlock_t		state_spinlock = SPIN_LOCK_UNLOCKED;
 
 nfs4_stateid zero_stateid =
 	{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
@@ -60,7 +61,7 @@ static LIST_HEAD(nfs4_clientid_list);
  * Since these are allocated/deallocated very rarely, we don't
  * bother putting them in a slab cache...
  */
-struct nfs4_client *
+static struct nfs4_client *
 nfs4_alloc_client(struct in_addr *addr)
 {
 	struct nfs4_client *clp;
@@ -70,6 +71,7 @@ nfs4_alloc_client(struct in_addr *addr)
 		memcpy(&clp->cl_addr, addr, sizeof(clp->cl_addr));
 		init_rwsem(&clp->cl_sem);
 		INIT_LIST_HEAD(&clp->cl_state_owners);
+		INIT_LIST_HEAD(&clp->cl_unused);
 		spin_lock_init(&clp->cl_lock);
 		atomic_set(&clp->cl_count, 1);
 		clp->cl_state = NFS4CLNT_NEW;
@@ -77,10 +79,19 @@ nfs4_alloc_client(struct in_addr *addr)
 	return clp;
 }
 
-void
+static void
 nfs4_free_client(struct nfs4_client *clp)
 {
-	BUG_ON(!clp);
+	struct nfs4_state_owner *sp;
+
+	while (!list_empty(&clp->cl_unused)) {
+		sp = list_entry(clp->cl_unused.next,
+				struct nfs4_state_owner,
+				so_list);
+		list_del(&sp->so_list);
+		kfree(sp);
+	}
+	BUG_ON(!list_empty(&clp->cl_state_owners));
 	kfree(clp);
 }
 
@@ -120,149 +131,224 @@ nfs4_put_client(struct nfs4_client *clp)
 static inline u32
 nfs4_alloc_lockowner_id(struct nfs4_client *clp)
 {
-	u32 res;
+	return clp->cl_lockowner_id ++;
+}
 
-	spin_lock(&state_spinlock);
-	res = clp->cl_lockowner_id ++;
-	spin_unlock(&state_spinlock);
-	return res;
+static struct nfs4_state_owner *
+nfs4_client_grab_unused(struct nfs4_client *clp, struct rpc_cred *cred)
+{
+	struct nfs4_state_owner *sp = NULL;
+
+	if (!list_empty(&clp->cl_unused)) {
+		sp = list_entry(clp->cl_unused.next, struct nfs4_state_owner, so_list);
+		atomic_inc(&sp->so_count);
+		sp->so_cred = cred;
+		list_move(&sp->so_list, &clp->cl_state_owners);
+		clp->cl_nunused--;
+	}
+	return sp;
 }
 
 /*
- * nfs4_get_state_owner(): this is called on the OPEN or CREATE path to
- * obtain a new state_owner.
- *
- * There are three state_owners (open_owner4 in rfc3010) per inode,
- * one for each possible combination of share lock access. Since
- * Linux does not support the deny access type, there are
- * three (not 9) referenced by the nfs_inode:
- *
- * O_WRONLY: inode->wo_owner
- * O_RDONLY: inode->ro_owner
- * O_RDWR:   inode->rw_owner
- *
- * We create a new state_owner the first time a file is OPENed with
- * one of the above shares. All other OPENs with a similar
- * share use the single stateid associated with the inode.
+ * nfs4_alloc_state_owner(): this is called on the OPEN or CREATE path to
+ * create a new state_owner.
  *
  */
-struct nfs4_state_owner *
-nfs4_get_state_owner(struct inode *dir)
+static struct nfs4_state_owner *
+nfs4_alloc_state_owner(void)
 {
-	struct nfs4_client *clp;
 	struct nfs4_state_owner *sp;
 
 	sp = kmalloc(sizeof(*sp),GFP_KERNEL);
 	if (!sp)
 		return NULL;
-	clp = (NFS_SB(dir->i_sb))->nfs4_state;
-	BUG_ON(!clp);
 	init_MUTEX(&sp->so_sema);
 	sp->so_seqid = 0;                 /* arbitrary */
-	memset(sp->so_stateid, 0, sizeof(nfs4_stateid));
-	sp->so_id = nfs4_alloc_lockowner_id(clp);
+	INIT_LIST_HEAD(&sp->so_states);
+	atomic_set(&sp->so_count, 1);
 	return sp;
 }
 
-/*
- * Called for each non-null inode state_owner in nfs_clear_inode, 
- * or if nfs4_do_open fails.
- */
-void
-nfs4_put_state_owner(struct inode *inode, struct nfs4_state_owner *sp)
+struct nfs4_state_owner *
+nfs4_get_state_owner(struct nfs_server *server, struct rpc_cred *cred)
 {
+	struct nfs4_client *clp = server->nfs4_state;
+	struct nfs4_state_owner *sp, *new;
+
+	get_rpccred(cred);
+	new = nfs4_alloc_state_owner();
+	spin_lock(&clp->cl_lock);
+	sp = nfs4_client_grab_unused(clp, cred);
+	if (sp == NULL && new != NULL) {
+		list_add(&new->so_list, &clp->cl_state_owners);
+		new->so_client = clp;
+		new->so_id = nfs4_alloc_lockowner_id(clp);
+		new->so_cred = cred;
+		sp = new;
+		new = NULL;
+	}
+	spin_unlock(&clp->cl_lock);
+	if (new)
+		kfree(new);
 	if (!sp)
-		return;
-	if (sp->so_flags & O_ACCMODE)
-		nfs4_do_close(inode, sp);
-        kfree(sp);
+		put_rpccred(cred);
+	return sp;
 }
 
-/*
-* Called with sp->so_sema held.
-*
-* Increment the seqid if the OPEN/OPEN_DOWNGRADE/CLOSE succeeded, or
-* failed with a seqid incrementing error -
-* see comments nfs_fs.h:seqid_mutating_error()
-*/
 void
-nfs4_increment_seqid(u32 status, struct nfs4_state_owner *sp)
+nfs4_put_state_owner(struct nfs4_state_owner *sp)
 {
-	if (status == NFS_OK || seqid_mutating_err(status))
-		sp->so_seqid++;
+	struct nfs4_client *clp = sp->so_client;
+	struct rpc_cred *cred = sp->so_cred;
+
+	if (!atomic_dec_and_lock(&sp->so_count, &clp->cl_lock))
+		return;
+	if (clp->cl_nunused >= OPENOWNER_POOL_SIZE)
+		goto out_free;
+	list_move(&sp->so_list, &clp->cl_unused);
+	clp->cl_nunused++;
+	spin_unlock(&clp->cl_lock);
+	put_rpccred(cred);
+	cred = NULL;
+	return;
+out_free:
+	list_del(&sp->so_list);
+	spin_unlock(&clp->cl_lock);
+	put_rpccred(cred);
+	kfree(sp);
 }
 
-/*
-* Called by nfs4_proc_open to set the appropriate stateid
-*/
-int
-nfs4_set_inode_share(struct inode * inode, struct nfs4_state_owner *sp, unsigned int open_flags)
+static struct nfs4_state *
+nfs4_alloc_open_state(void)
+{
+	struct nfs4_state *state;
+
+	state = kmalloc(sizeof(*state), GFP_KERNEL);
+	if (!state)
+		return NULL;
+	state->pid = current->pid;
+	state->state = 0;
+	memset(state->stateid, 0, sizeof(state->stateid));
+	atomic_set(&state->count, 1);
+	return state;
+}
+
+static struct nfs4_state *
+__nfs4_find_state_bypid(struct inode *inode, pid_t pid)
 {
 	struct nfs_inode *nfsi = NFS_I(inode);
+	struct nfs4_state *state;
 
-	switch (open_flags & O_ACCMODE) {
-		case O_RDONLY:
-			if (!nfsi->ro_owner) {
-				nfsi->ro_owner = sp;
-				return 0;
-			}
-			break;
-		case O_WRONLY:
-			if (!nfsi->wo_owner) {
-				nfsi->wo_owner = sp;
-				return 0;
-			}
-			break;
-		case O_RDWR:
-			if (!nfsi->rw_owner) {
-				nfsi->rw_owner = sp;
-				return 0;
-			}
+	list_for_each_entry(state, &nfsi->open_states, inode_states) {
+		if (state->pid == pid) {
+			atomic_inc(&state->count);
+			return state;
+		}
 	}
-	return -EBUSY;
+	return NULL;
 }
 
-/*
-* Boolean test to determine if an OPEN call goes on the wire.
-*
-* Called by nfs4_proc_open.
-*/
-int
-nfs4_test_state_owner(struct inode *inode, unsigned int open_flags)
+static struct nfs4_state *
+__nfs4_find_state_byowner(struct inode *inode, struct nfs4_state_owner *owner)
 {
 	struct nfs_inode *nfsi = NFS_I(inode);
+	struct nfs4_state *state;
 
-	switch (open_flags & O_ACCMODE) {
-		case O_RDONLY:
-			if(nfsi->ro_owner)
-				return 0;
-			break;
-		case O_WRONLY:
-			if(nfsi->wo_owner)
-				return 0;
-			break;
-		case O_RDWR:
-			if(nfsi->rw_owner)
-				return 0;
-        }
-        return 1;
+	list_for_each_entry(state, &nfsi->open_states, inode_states) {
+		if (state->owner == owner) {
+			atomic_inc(&state->count);
+			return state;
+		}
+	}
+	return NULL;
 }
 
-struct nfs4_state_owner *
-nfs4_get_inode_share(struct inode * inode, unsigned int open_flags)
+struct nfs4_state *
+nfs4_find_state_bypid(struct inode *inode, pid_t pid)
+{
+	struct nfs_inode *nfsi = NFS_I(inode);
+	struct nfs4_state *state;
+
+	spin_lock(&inode->i_lock);
+	state = __nfs4_find_state_bypid(inode, pid);
+	/* Add the state to the tail of the inode's list */
+	if (state)
+		list_move_tail(&state->inode_states, &nfsi->open_states);
+	spin_unlock(&inode->i_lock);
+	return state;
+}
+
+static void
+nfs4_free_open_state(struct nfs4_state *state)
 {
+	kfree(state);
+}
+
+struct nfs4_state *
+nfs4_get_open_state(struct inode *inode, struct nfs4_state_owner *owner)
+{
+	struct nfs4_state *state, *new;
 	struct nfs_inode *nfsi = NFS_I(inode);
 
-	switch (open_flags & O_ACCMODE) {
-		case O_RDONLY:
-			return nfsi->ro_owner;
-		case O_WRONLY:
-			return nfsi->wo_owner;
-		case O_RDWR:
-			return nfsi->rw_owner;
+	spin_lock(&inode->i_lock);
+	state = __nfs4_find_state_byowner(inode, owner);
+	spin_unlock(&inode->i_lock);
+	if (state)
+		goto out;
+	new = nfs4_alloc_open_state();
+	spin_lock(&inode->i_lock);
+	state = __nfs4_find_state_byowner(inode, owner);
+	if (state == NULL && new != NULL) {
+		state = new;
+		/* Caller *must* be holding owner->so_sem */
+		list_add(&state->open_states, &owner->so_states);
+		state->owner = owner;
+		atomic_inc(&owner->so_count);
+		list_add(&state->inode_states, &nfsi->open_states);
+		state->inode = inode;
+		atomic_inc(&inode->i_count);
+		spin_unlock(&inode->i_lock);
+	} else {
+		spin_unlock(&inode->i_lock);
+		if (new)
+			nfs4_free_open_state(new);
 	}
-	/* Duh gcc warning if we don't... */
-	return NULL;
+out:
+	return state;
+}
+
+void
+nfs4_put_open_state(struct nfs4_state *state)
+{
+	struct inode *inode = state->inode;
+	struct nfs4_state_owner *owner = state->owner;
+
+	if (!atomic_dec_and_lock(&state->count, &inode->i_lock))
+		return;
+	list_del(&state->inode_states);
+	spin_unlock(&inode->i_lock);
+	down(&owner->so_sema);
+	list_del(&state->open_states);
+	if (state->state != 0)
+		nfs4_do_close(inode, state);
+	up(&owner->so_sema);
+	iput(inode);
+	nfs4_free_open_state(state);
+	nfs4_put_state_owner(owner);
+}
+
+/*
+* Called with sp->so_sema held.
+*
+* Increment the seqid if the OPEN/OPEN_DOWNGRADE/CLOSE succeeded, or
+* failed with a seqid incrementing error -
+* see comments nfs_fs.h:seqid_mutating_error()
+*/
+void
+nfs4_increment_seqid(u32 status, struct nfs4_state_owner *sp)
+{
+	if (status == NFS_OK || seqid_mutating_err(status))
+		sp->so_seqid++;
 }
 
 /*

fs/nfs/nfs4xdr.c

@@ -553,14 +553,17 @@ encode_open(struct xdr_stream *xdr, struct nfs_openargs *arg)
 	WRITE32(OP_OPEN);
 	WRITE32(arg->seqid);
 	switch (arg->share_access) {
-		case O_RDONLY:
+		case FMODE_READ:
 			WRITE32(NFS4_SHARE_ACCESS_READ);
 			break;
-		case O_WRONLY:
+		case FMODE_WRITE:
 			WRITE32(NFS4_SHARE_ACCESS_WRITE);
 			break;
-		case O_RDWR:
+		case FMODE_READ|FMODE_WRITE:
 			WRITE32(NFS4_SHARE_ACCESS_BOTH);
+			break;
+		default:
+			BUG();
 	}
 	WRITE32(0);                  /* for linux, share_deny = 0 always */
 	WRITE64(arg->clientid);

fs/nfs/pagelist.c

@@ -49,7 +49,7 @@ nfs_page_free(struct nfs_page *p)
 
 /**
  * nfs_create_request - Create an NFS read/write request.
- * @cred: RPC credential to use
+ * @file: file descriptor to use
  * @inode: inode to which the request is attached
  * @page: page to write
  * @offset: starting offset within the page for the write
@@ -62,7 +62,7 @@ nfs_page_free(struct nfs_page *p)
  * User should ensure it is safe to sleep in this function.
  */
 struct nfs_page *
-nfs_create_request(struct rpc_cred *cred, struct inode *inode,
+nfs_create_request(struct file *file, struct inode *inode,
 		   struct page *page,
 		   unsigned int offset, unsigned int count)
 {
@@ -93,11 +93,9 @@ nfs_create_request(struct rpc_cred *cred, struct inode *inode,
 	req->wb_offset  = offset;
 	req->wb_pgbase	= offset;
 	req->wb_bytes   = count;
-
-	if (cred)
-		req->wb_cred = get_rpccred(cred);
 	req->wb_inode   = inode;
 	req->wb_count   = 1;
+	server->rpc_ops->request_init(req, file);
 
 	return req;
 }
@@ -111,6 +109,8 @@ nfs_create_request(struct rpc_cred *cred, struct inode *inode,
  */
 void nfs_clear_request(struct nfs_page *req)
 {
+	if (req->wb_state)
+		req->wb_state = NULL;
 	/* Release struct file or cached credential */
 	if (req->wb_file) {
 		fput(req->wb_file);

fs/nfs/proc.c

@@ -60,6 +60,18 @@ nfs_write_refresh_inode(struct inode *inode, struct nfs_fattr *fattr)
 	nfs_refresh_inode(inode, fattr);
 }
 
+static struct rpc_cred *
+nfs_cred(struct inode *inode, struct file *filp)
+{
+	struct rpc_cred *cred = NULL;
+
+	if (filp)
+		cred = (struct rpc_cred *)filp->private_data;
+	if (!cred)
+		cred = NFS_I(inode)->mm_cred;
+	return cred;
+}
+
 /*
  * Bare-bones access to getattr: this is for nfs_read_super.
  */
@@ -149,7 +161,7 @@ nfs_proc_readlink(struct inode *inode, struct page *page)
 }
 
 static int
-nfs_proc_read(struct nfs_read_data *rdata)
+nfs_proc_read(struct nfs_read_data *rdata, struct file *filp)
 {
 	int			flags = rdata->flags;
 	struct inode *		inode = rdata->inode;
@@ -158,13 +170,13 @@ nfs_proc_read(struct nfs_read_data *rdata)
 		.rpc_proc	= &nfs_procedures[NFSPROC_READ],
 		.rpc_argp	= &rdata->args,
 		.rpc_resp	= &rdata->res,
-		.rpc_cred	= rdata->cred,
 	};
 	int			status;
 
 	dprintk("NFS call  read %d @ %Ld\n", rdata->args.count,
 			(long long) rdata->args.offset);
 	fattr->valid = 0;
+	msg.rpc_cred = nfs_cred(inode, filp);
 	status = rpc_call_sync(NFS_CLIENT(inode), &msg, flags);
 
 	if (status >= 0)
@@ -174,7 +186,7 @@ nfs_proc_read(struct nfs_read_data *rdata)
 }
 
 static int
-nfs_proc_write(struct nfs_write_data *wdata)
+nfs_proc_write(struct nfs_write_data *wdata, struct file *filp)
 {
 	int			flags = wdata->flags;
 	struct inode *		inode = wdata->inode;
@@ -183,13 +195,13 @@ nfs_proc_write(struct nfs_write_data *wdata)
 		.rpc_proc	= &nfs_procedures[NFSPROC_WRITE],
 		.rpc_argp	= &wdata->args,
 		.rpc_resp	= &wdata->res,
-		.rpc_cred	= wdata->cred
 	};
 	int			status;
 
 	dprintk("NFS call  write %d @ %Ld\n", wdata->args.count,
 			(long long) wdata->args.offset);
 	fattr->valid = 0;
+	msg.rpc_cred = nfs_cred(inode, filp);
 	status = rpc_call_sync(NFS_CLIENT(inode), &msg, flags);
 	if (status >= 0) {
 		nfs_write_refresh_inode(inode, fattr);
@@ -200,10 +212,12 @@ nfs_proc_write(struct nfs_write_data *wdata)
 	return status < 0? status : wdata->res.count;
 }
 
-static int
+static struct inode *
 nfs_proc_create(struct inode *dir, struct qstr *name, struct iattr *sattr,
-		int flags, struct nfs_fh *fhandle, struct nfs_fattr *fattr)
+		int flags)
 {
+	struct nfs_fh		fhandle;
+	struct nfs_fattr	fattr;
 	struct nfs_createargs	arg = {
 		.fh		= NFS_FH(dir),
 		.name		= name->name,
@@ -211,16 +225,23 @@ nfs_proc_create(struct inode *dir, struct qstr *name, struct iattr *sattr,
 		.sattr		= sattr
 	};
 	struct nfs_diropok	res = {
-		.fh		= fhandle,
-		.fattr		= fattr
+		.fh		= &fhandle,
+		.fattr		= &fattr
 	};
 	int			status;
 
-	fattr->valid = 0;
+	fattr.valid = 0;
 	dprintk("NFS call  create %s\n", name->name);
 	status = rpc_call(NFS_CLIENT(dir), NFSPROC_CREATE, &arg, &res, 0);
 	dprintk("NFS reply create: %d\n", status);
-	return status;
+	if (status == 0) {
+		struct inode *inode;
+		inode = nfs_fhget(dir->i_sb, &fhandle, &fattr);
+		if (inode)
+			return inode;
+		status = -ENOMEM;
+	}
+	return ERR_PTR(status);
 }
 
 /*
@@ -611,6 +632,28 @@ nfs_proc_commit_setup(struct nfs_write_data *data, u64 start, u32 len, int how)
 	BUG();
 }
 
+/*
+ * Set up the nfspage struct with the right credentials
+ */
+static void
+nfs_request_init(struct nfs_page *req, struct file *filp)
+{
+	req->wb_cred = get_rpccred(nfs_cred(req->wb_inode, filp));
+}
+
+static int
+nfs_request_compatible(struct nfs_page *req, struct file *filp, struct page *page)
+{
+	if (req->wb_file != filp)
+		return 0;
+	if (req->wb_page != page)
+		return 0;
+	if (req->wb_cred != nfs_file_cred(filp))
+		return 0;
+	return 1;
+}
+
+
 struct nfs_rpc_ops	nfs_v2_clientops = {
 	.version	= 2,		       /* protocol version */
 	.getroot	= nfs_proc_get_root,
@@ -640,4 +683,8 @@ struct nfs_rpc_ops	nfs_v2_clientops = {
 	.read_setup	= nfs_proc_read_setup,
 	.write_setup	= nfs_proc_write_setup,
 	.commit_setup	= nfs_proc_commit_setup,
+	.file_open	= nfs_open,
+	.file_release	= nfs_release,
+	.request_init	= nfs_request_init,
+	.request_compatible = nfs_request_compatible,
 };

fs/nfs/read.c

@@ -89,9 +89,6 @@ nfs_readpage_sync(struct file *file, struct inode *inode, struct page *page)
 
 	dprintk("NFS: nfs_readpage_sync(%p)\n", page);
 
-	if (file)
-		rdata.cred = nfs_file_cred(file);
-
 	/*
 	 * This works now because the socket layer never tries to DMA
 	 * into this buffer directly.
@@ -110,7 +107,7 @@ nfs_readpage_sync(struct file *file, struct inode *inode, struct page *page)
 			rdata.args.count);
 
 		lock_kernel();
-		result = NFS_PROTO(inode)->read(&rdata);
+		result = NFS_PROTO(inode)->read(&rdata, file);
 		unlock_kernel();
 
 		/*
@@ -146,7 +143,7 @@ nfs_readpage_async(struct file *file, struct inode *inode, struct page *page)
 	LIST_HEAD(one_request);
 	struct nfs_page	*new;
 
-	new = nfs_create_request(nfs_file_cred(file), inode, page, 0, PAGE_CACHE_SIZE);
+	new = nfs_create_request(file, inode, page, 0, PAGE_CACHE_SIZE);
 	if (IS_ERR(new)) {
 		unlock_page(page);
 		return PTR_ERR(new);
@@ -363,8 +360,7 @@ readpage_async_filler(void *data, struct page *page)
 	struct nfs_page *new;
 
 	nfs_wb_page(inode, page);
-	new = nfs_create_request(nfs_file_cred(desc->filp), inode, page,
-				0, PAGE_CACHE_SIZE);
+	new = nfs_create_request(desc->filp, inode, page, 0, PAGE_CACHE_SIZE);
 	if (IS_ERR(new)) {
 			SetPageError(page);
 			unlock_page(page);

fs/nfs/write.c

@@ -152,11 +152,6 @@ nfs_writepage_sync(struct file *file, struct inode *inode, struct page *page,
 		},
 	};
 
-	if (file)
-		wdata.cred = get_rpccred(nfs_file_cred(file));
-	if (!wdata.cred)
-		wdata.cred = get_rpccred(NFS_I(inode)->mm_cred);
-
 	dprintk("NFS:      nfs_writepage_sync(%s/%Ld %d@%Ld)\n",
 		inode->i_sb->s_id,
 		(long long)NFS_FILEID(inode),
@@ -167,7 +162,7 @@ nfs_writepage_sync(struct file *file, struct inode *inode, struct page *page,
 			wdata.args.count = count;
 		wdata.args.offset = page_offset(page) + wdata.args.pgbase;
 
-		result = NFS_PROTO(inode)->write(&wdata);
+		result = NFS_PROTO(inode)->write(&wdata, file);
 
 		if (result < 0) {
 			/* Must mark the page invalid after I/O error */
@@ -213,8 +208,6 @@ nfs_writepage_async(struct file *file, struct inode *inode, struct page *page,
 	status = (IS_ERR(req)) ? PTR_ERR(req) : 0;
 	if (status < 0)
 		goto out;
-	if (!req->wb_cred)
-		req->wb_cred = get_rpccred(NFS_I(inode)->mm_cred);
 	nfs_unlock_request(req);
 	nfs_strategy(inode);
 	end = ((loff_t)page->index<<PAGE_CACHE_SHIFT) + (loff_t)(offset + count);
@@ -556,7 +549,7 @@ nfs_update_request(struct file* file, struct inode *inode, struct page *page,
 		}
 		spin_unlock(&nfs_wreq_lock);
 
-		new = nfs_create_request(nfs_file_cred(file), inode, page, offset, bytes);
+		new = nfs_create_request(file, inode, page, offset, bytes);
 		if (IS_ERR(new))
 			return new;
 		if (file) {
@@ -637,7 +630,6 @@ int
 nfs_flush_incompatible(struct file *file, struct page *page)
 {
 	struct inode	*inode = page->mapping->host;
-	struct rpc_cred	*cred = nfs_file_cred(file);
 	struct nfs_page	*req;
 	int		status = 0;
 	/*
@@ -650,7 +642,7 @@ nfs_flush_incompatible(struct file *file, struct page *page)
 	 */
 	req = nfs_find_request(inode, page->index);
 	if (req) {
-		if (req->wb_file != file || req->wb_cred != cred || req->wb_page != page)
+		if (!NFS_PROTO(inode)->request_compatible(req, file, page))
 			status = nfs_wb_page(inode, page);
 		nfs_release_request(req);
 	}

include/linux/nfs_fs.h

@@ -27,7 +27,6 @@
 #include <linux/nfs2.h>
 #include <linux/nfs3.h>
 #include <linux/nfs4.h>
-#include <linux/nfs_page.h>
 #include <linux/nfs_xdr.h>
 
 /*
@@ -158,9 +157,7 @@ struct nfs_inode {
 
 #ifdef CONFIG_NFS_V4
         /* NFSv4 state */
-	struct nfs4_state_owner   *ro_owner;
-	struct nfs4_state_owner   *wo_owner;
-	struct nfs4_state_owner   *rw_owner;
+	struct list_head	open_states;
 #endif /* CONFIG_NFS_V4*/
 
 	struct inode		vfs_inode;
@@ -229,12 +226,6 @@ loff_t page_offset(struct page *page)
 	return ((loff_t)page->index) << PAGE_CACHE_SHIFT;
 }
 
-static inline
-loff_t req_offset(struct nfs_page *req)
-{
-	return (((loff_t)req->wb_index) << PAGE_CACHE_SHIFT) + req->wb_offset;
-}
-
 /*
  * linux/fs/nfs/inode.c
  */
@@ -244,6 +235,7 @@ extern struct inode *nfs_fhget(struct super_block *, struct nfs_fh *,
 extern int __nfs_refresh_inode(struct inode *, struct nfs_fattr *);
 extern int nfs_getattr(struct vfsmount *, struct dentry *, struct kstat *);
 extern int nfs_permission(struct inode *, int, struct nameidata *);
+extern void nfs_set_mmcred(struct inode *, struct rpc_cred *);
 extern int nfs_open(struct inode *, struct file *);
 extern int nfs_release(struct inode *, struct file *);
 extern int __nfs_revalidate_inode(struct nfs_server *, struct inode *);
@@ -494,29 +486,60 @@ struct nfs4_client {
 	struct rw_semaphore	cl_sem;
 
 	struct list_head	cl_state_owners;
+	struct list_head	cl_unused;
+	int			cl_nunused;
 	spinlock_t		cl_lock;
 	atomic_t		cl_count;
 };
 
 /*
-* The ->so_sema is held during all state_owner seqid-mutating operations:
-* OPEN, OPEN_DOWNGRADE, and CLOSE.
-* Its purpose is to properly serialize so_seqid, as mandated by
-* the protocol.
-*/
+ * NFS4 state_owners and lock_owners are simply labels for ordered
+ * sequences of RPC calls. Their sole purpose is to provide once-only
+ * semantics by allowing the server to identify replayed requests.
+ *
+ * The ->so_sema is held during all state_owner seqid-mutating operations:
+ * OPEN, OPEN_DOWNGRADE, and CLOSE. Its purpose is to properly serialize
+ * so_seqid.
+ */
 struct nfs4_state_owner {
 	struct list_head     so_list;	 /* per-clientid list of state_owners */
+	struct nfs4_client   *so_client;
 	u32                  so_id;      /* 32-bit identifier, unique */
 	struct semaphore     so_sema;
 	u32                  so_seqid;   /* protected by so_sema */
-	nfs4_stateid         so_stateid; /* protected by so_sema */
 	unsigned int         so_flags;   /* protected by so_sema */
+	atomic_t	     so_count;
+
+	struct rpc_cred	     *so_cred;	 /* Associated cred */
+	struct list_head     so_states;
+};
+
+/*
+ * struct nfs4_state maintains the client-side state for a given
+ * (state_owner,inode) tuple.
+ *
+ * In order to know when to OPEN_DOWNGRADE or CLOSE the state on the server,
+ * we need to know how many files are open for reading or writing on a
+ * given inode. This information too is stored here.
+ */
+struct nfs4_state {
+	struct list_head open_states;	/* List of states for the same state_owner */
+	struct list_head inode_states;	/* List of states for the same inode */
+
+	struct nfs4_state_owner *owner;	/* Pointer to the open owner */
+	struct inode *inode;		/* Pointer to the inode */
+	pid_t pid;			/* Thread that called OPEN */
+
+	nfs4_stateid stateid;
+
+	int state;			/* State on the server (R,W, or RW) */
+	atomic_t count;
 };
 
 
 /* nfs4proc.c */
 extern int nfs4_proc_renew(struct nfs_server *server);
-extern int nfs4_do_close(struct inode *inode, struct nfs4_state_owner *sp);
+extern int nfs4_do_close(struct inode *, struct nfs4_state *);
 
 /* nfs4renewd.c */
 extern int nfs4_init_renewd(struct nfs_server *server);
@@ -524,13 +547,11 @@ extern int nfs4_init_renewd(struct nfs_server *server);
 /* nfs4state.c */
 extern struct nfs4_client *nfs4_get_client(struct in_addr *);
 extern void nfs4_put_client(struct nfs4_client *clp);
-extern struct nfs4_state_owner * nfs4_get_state_owner(struct inode *inode);
-void nfs4_put_state_owner(struct inode *inode, struct nfs4_state_owner *sp);
-extern int nfs4_set_inode_share(struct inode * inode,
-                     struct nfs4_state_owner *sp, unsigned int flags);
+extern struct nfs4_state_owner * nfs4_get_state_owner(struct nfs_server *, struct rpc_cred *);
+extern void nfs4_put_state_owner(struct nfs4_state_owner *);
+extern struct nfs4_state * nfs4_get_open_state(struct inode *, struct nfs4_state_owner *);
+extern void nfs4_put_open_state(struct nfs4_state *);
 extern void nfs4_increment_seqid(u32 status, struct nfs4_state_owner *sp);
-extern int nfs4_test_state_owner(struct inode *inode, unsigned int open_flags);
-struct nfs4_state_owner * nfs4_get_inode_share(struct inode * inode, unsigned int open_flags);
 
 
 
@@ -561,6 +582,7 @@ destroy_nfsv4_state(struct nfs_server *server)
 #define create_nfsv4_state(server, data)  0
 #define destroy_nfsv4_state(server)       do { } while (0)
 #define nfs4_put_state_owner(inode, owner) do { } while (0)
+#define nfs4_put_open_state(state) do { } while (0)
 #endif
 
 #endif /* __KERNEL__ */

include/linux/nfs_page.h

@@ -11,7 +11,7 @@
 
 
 #include <linux/list.h>
-#include <linux/mm.h>
+#include <linux/pagemap.h>
 #include <linux/wait.h>
 #include <linux/nfs_fs_sb.h>
 #include <linux/sunrpc/auth.h>
@@ -28,6 +28,7 @@ struct nfs_page {
 	struct file		*wb_file;
 	struct inode		*wb_inode;
 	struct rpc_cred		*wb_cred;
+	struct nfs4_state	*wb_state;
 	struct page		*wb_page;	/* page to read in/write out */
 	wait_queue_head_t	wb_wait;	/* wait queue */
 	unsigned long		wb_index;	/* Offset >> PAGE_CACHE_SHIFT */
@@ -41,7 +42,7 @@ struct nfs_page {
 
 #define NFS_WBACK_BUSY(req)	(test_bit(PG_BUSY,&(req)->wb_flags))
 
-extern	struct nfs_page *nfs_create_request(struct rpc_cred *, struct inode *,
+extern	struct nfs_page *nfs_create_request(struct file *, struct inode *,
 					    struct page *,
 					    unsigned int, unsigned int);
 extern	void nfs_clear_request(struct nfs_page *req);
@@ -121,4 +122,10 @@ nfs_list_entry(struct list_head *head)
 	return list_entry(head, struct nfs_page, wb_list);
 }
 
+static inline
+loff_t req_offset(struct nfs_page *req)
+{
+	return (((loff_t)req->wb_index) << PAGE_CACHE_SHIFT) + req->wb_offset;
+}
+
 #endif /* _LINUX_NFS_PAGE_H */

include/linux/nfs_xdr.h

@@ -620,6 +620,8 @@ struct nfs_write_data {
 #endif
 };
 
+struct nfs_page;
+
 /*
  * RPC procedure vector for NFSv2/NFSv3 demuxing
  */
@@ -635,12 +637,12 @@ struct nfs_rpc_ops {
 			    struct nfs_fh *, struct nfs_fattr *);
 	int	(*access)  (struct inode *, struct rpc_cred *, int);
 	int	(*readlink)(struct inode *, struct page *);
-	int	(*read)    (struct nfs_read_data *);
-	int	(*write)   (struct nfs_write_data *);
+	int	(*read)    (struct nfs_read_data *, struct file *);
+	int	(*write)   (struct nfs_write_data *, struct file *);
 	int	(*commit)  (struct inode *, struct nfs_fattr *,
 			    unsigned long, unsigned int);
-	int	(*create)  (struct inode *, struct qstr *, struct iattr *,
-			    int, struct nfs_fh *, struct nfs_fattr *);
+	struct inode *	(*create)  (struct inode *, struct qstr *,
+			    struct iattr *, int);
 	int	(*remove)  (struct inode *, struct qstr *);
 	int	(*unlink_setup)  (struct rpc_message *,
 			    struct dentry *, struct qstr *);
@@ -669,6 +671,9 @@ struct nfs_rpc_ops {
 	void	(*write_setup)  (struct nfs_write_data *, unsigned int count, int how);
 	void	(*commit_setup) (struct nfs_write_data *, u64 start, u32 len, int how);
 	int	(*file_open)   (struct inode *, struct file *);
+	int	(*file_release) (struct inode *, struct file *);
+	void	(*request_init)(struct nfs_page *, struct file *);
+	int	(*request_compatible)(struct nfs_page *, struct file *, struct page *);
 };
 
 /*