[PATCH] NFSv4 server OPEN_CONFIRM

From: "William A.(Andy) Adamson" <andros@citi.umich.edu>

Add OPEN_CONFIRM functionality to the nfsv4 server, including a
preprocess_seqid_op function that will be used in subsequent patches.

fs/nfsd/nfs4proc.c

@@ -140,10 +140,6 @@ nfsd4_open(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfsd4_open
 	status = nfsd4_process_open2(rqstp, current_fh, open);
 	if (status)
 		return status;
-	/*
-	 * To finish the open response, we just need to set the rflags.
-	 */
-	open->op_rflags = 0;
 	return 0;
 }
 
@@ -609,6 +605,9 @@ nfsd4_proc_compound(struct svc_rqst *rqstp,
 		case OP_OPEN:
 			op->status = nfsd4_open(rqstp, &current_fh, &op->u.open);
 			break;
+		case OP_OPEN_CONFIRM:
+			op->status = nfsd4_open_confirm(rqstp, &current_fh, &op->u.open_confirm);
+			break;
 		case OP_PUTFH:
 			op->status = nfsd4_putfh(rqstp, &current_fh, &op->u.putfh);
 			break;

fs/nfsd/nfs4state.c

@@ -56,6 +56,8 @@ static u32 current_clientid = 1;
 static u32 current_ownerid = 0;
 static u32 current_fileid = 0;
 static u32 nfs4_init = 0;
+stateid_t zerostateid;             /* bits all 0 */
+stateid_t onestateid;              /* bits all 1 */
 
 /* debug counters */
 u32 list_add_perfile = 0; 
@@ -136,7 +138,8 @@ STALE_CLIENTID(clientid_t *clid)
 {
 	if (clid->cl_boot == boot_time)
 		return 0;
-	printk("NFSD stale clientid (%08x/%08x)\n", clid->cl_boot, clid->cl_id);
+	dprintk("NFSD stale clientid (%08x/%08x)\n", 
+			clid->cl_boot, clid->cl_id);
 	return 1;
 }
 
@@ -641,25 +644,38 @@ nfsd4_setclientid_confirm(struct svc_rqst *rqstp, struct nfsd4_setclientid_confi
  * Open owner state (share locks)
  */
 
+#define ZERO_STATEID(stateid)       (!memcmp((stateid), &zerostateid, sizeof(stateid_t)))
+#define ONE_STATEID(stateid)        (!memcmp((stateid), &onestateid, sizeof(stateid_t)))
+
 /* hash tables for nfs4_stateowner */
 #define OWNER_HASH_BITS              8
 #define OWNER_HASH_SIZE             (1 << OWNER_HASH_BITS)
 #define OWNER_HASH_MASK             (OWNER_HASH_SIZE - 1)
 
+#define ownerid_hashval(id) \
+        ((id) & OWNER_HASH_MASK)
 #define ownerstr_hashval(clientid, ownername) \
         (((clientid) + opaque_hashval((ownername.data), (ownername.len))) & OWNER_HASH_MASK)
 
+static struct list_head	ownerid_hashtbl[OWNER_HASH_SIZE];
 static struct list_head	ownerstr_hashtbl[OWNER_HASH_SIZE];
 
 /* hash table for nfs4_file */
 #define FILE_HASH_BITS                   8
 #define FILE_HASH_SIZE                  (1 << FILE_HASH_BITS)
 #define FILE_HASH_MASK                  (FILE_HASH_SIZE - 1)
+/* hash table for (open)nfs4_stateid */
+#define OPENSTATEID_HASH_BITS              10
+#define OPENSTATEID_HASH_SIZE              (1 << OPENSTATEID_HASH_BITS)
+#define OPENSTATEID_HASH_MASK              (OPENSTATEID_HASH_SIZE - 1)
 
 #define file_hashval(x) \
         ((unsigned int)((x)->dev + (x)->ino + (x)->generation) & FILE_HASH_MASK)
+#define openstateid_hashval(owner_id, file_id)  \
+        (((owner_id) + (file_id)) & OPENSTATEID_HASH_MASK)
 
 static struct list_head file_hashtbl[FILE_HASH_SIZE];
+static struct list_head openstateid_hashtbl[OPENSTATEID_HASH_SIZE];
 
 /* OPEN Share state helper functions */
 static inline struct nfs4_file *
@@ -724,20 +740,23 @@ free_stateowner(struct nfs4_stateowner *sop) {
 static struct nfs4_stateowner *
 alloc_init_stateowner(unsigned int strhashval, struct nfs4_client *clp, struct nfsd4_open *open) {
 	struct nfs4_stateowner *sop;
+	unsigned int idhashval;
 
 	if (!(sop = alloc_stateowner(&open->op_owner)))
 		return (struct nfs4_stateowner *)NULL;
+	idhashval = ownerid_hashval(current_ownerid);
+	INIT_LIST_HEAD(&sop->so_idhash);
 	INIT_LIST_HEAD(&sop->so_strhash);
 	INIT_LIST_HEAD(&sop->so_perclient);
 	INIT_LIST_HEAD(&sop->so_peropenstate);
+	list_add(&sop->so_idhash, &ownerid_hashtbl[idhashval]);
 	list_add(&sop->so_strhash, &ownerstr_hashtbl[strhashval]);
 	list_add(&sop->so_perclient, &clp->cl_perclient);
 	add_perclient++;
 	sop->so_id = current_ownerid++;
 	sop->so_client = clp;
 	sop->so_seqid = open->op_seqid;
-	/* until open_confirm is coded, pretend it happened! */
-	sop->so_confirmed = 1;
+	sop->so_confirmed = 0;
 	return sop;
 }
 
@@ -746,6 +765,7 @@ release_stateowner(struct nfs4_stateowner *sop)
 {
 	struct nfs4_stateid *stp;
 
+	list_del_init(&sop->so_idhash);
 	list_del_init(&sop->so_strhash);
 	list_del_init(&sop->so_perclient);
 	del_perclient++;
@@ -759,9 +779,12 @@ release_stateowner(struct nfs4_stateowner *sop)
 
 static inline void
 init_stateid(struct nfs4_stateid *stp, struct nfs4_file *fp, struct nfs4_stateowner *sop, struct nfsd4_open *open) {
+	unsigned int hashval = openstateid_hashval(sop->so_id, fp->fi_id);
 
+	INIT_LIST_HEAD(&stp->st_hash);
 	INIT_LIST_HEAD(&stp->st_peropenstate);
 	INIT_LIST_HEAD(&stp->st_perfile);
+	list_add(&stp->st_hash, &openstateid_hashtbl[hashval]);
 	list_add(&stp->st_peropenstate, &sop->so_peropenstate);
 	list_add_perfile++;
 	list_add(&stp->st_perfile, &fp->fi_perfile);
@@ -778,6 +801,7 @@ init_stateid(struct nfs4_stateid *stp, struct nfs4_file *fp, struct nfs4_stateow
 static void
 release_stateid(struct nfs4_stateid *stp) {
 
+	list_del_init(&stp->st_hash);
 	list_del_perfile++;
 	list_del_init(&stp->st_perfile);
 	list_del_init(&stp->st_peropenstate);
@@ -1063,6 +1087,13 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf
 	open->op_delegate_type = NFS4_OPEN_DELEGATE_NONE;
 	status = nfs_ok;
 out:
+	/*
+	* To finish the open response, we just need to set the rflags.
+	*/
+	open->op_rflags = 0;
+	if (!open->op_stateowner->so_confirmed)
+		open->op_rflags |= NFS4_OPEN_RESULT_CONFIRM;
+
 	up(&client_sema); /*XXX need finer grained locking */
 	return status;
 out_free:
@@ -1154,6 +1185,203 @@ laundromat_main(void *not_used)
 	schedule_delayed_work(&laundromat_work, t*HZ);
 }
 
+/* search openstateid_hashtbl[] for stateid */
+struct nfs4_stateid *
+find_stateid(stateid_t *stid)
+{
+	struct list_head *pos, *next;
+	struct nfs4_stateid *local = NULL;
+	u32 st_id = stid->si_stateownerid;
+	u32 f_id = stid->si_fileid;
+	unsigned int hashval = openstateid_hashval(st_id, f_id);
+
+	list_for_each_safe(pos, next, &openstateid_hashtbl[hashval]) {
+		local = list_entry(pos, struct nfs4_stateid, st_hash);
+		if((local->st_stateid.si_stateownerid == st_id) &&
+		   (local->st_stateid.si_fileid == f_id))
+			return local;
+	}
+	return NULL;
+}
+
+/* search ownerid_hashtbl[] for stateid owner (stateid->si_stateownerid) */
+struct nfs4_stateowner *
+find_stateowner_id(u32 st_id) {
+	struct list_head *pos, *next;
+	struct nfs4_stateowner *local = NULL;
+	unsigned int hashval = ownerid_hashval(st_id);
+
+	list_for_each_safe(pos, next, &ownerid_hashtbl[hashval]) {
+		local = list_entry(pos, struct nfs4_stateowner, so_idhash);
+		if(local->so_id == st_id)
+			return local;
+	}
+	return NULL;
+}
+
+static inline int
+nfs4_check_fh(struct svc_fh *fhp, struct nfs4_stateid *stp)
+{
+	return (fhp->fh_dentry != stp->st_vfs_file.f_dentry);
+}
+
+static int
+STALE_STATEID(stateid_t *stateid)
+{
+	if (stateid->si_boot == boot_time)
+		return 0;
+	printk("NFSD: stale stateid (%08x/%08x/%08x/%08x)!\n",
+		stateid->si_boot, stateid->si_stateownerid, stateid->si_fileid,
+		stateid->si_generation);
+	return 1;
+}
+
+
+/* 
+ * Checks for sequence id mutating operations. 
+ *
+ * XXX need to code replay cache logic
+ */
+int
+nfs4_preprocess_seqid_op(struct svc_fh *current_fh, u32 seqid, stateid_t *stateid, int flags, struct nfs4_stateowner **sopp, struct nfs4_stateid **stpp)
+{
+	int status;
+	struct nfs4_stateid *stp;
+	struct nfs4_stateowner *sop;
+
+	dprintk("NFSD: preprocess_seqid_op: seqid=%d " 
+			"stateid = (%08x/%08x/%08x/%08x)\n", seqid,
+		stateid->si_boot, stateid->si_stateownerid, stateid->si_fileid,
+		stateid->si_generation);
+			        
+	*stpp = NULL;
+	*sopp = NULL;
+
+	status = nfserr_bad_stateid;
+	if (ZERO_STATEID(stateid) || ONE_STATEID(stateid)) {
+		printk("NFSD: preprocess_seqid_op: magic stateid!\n");
+		goto out;
+	}
+
+	status = nfserr_stale_stateid;
+	if (STALE_STATEID(stateid))
+		goto out;
+	/*
+	* We return BAD_STATEID if filehandle doesn't match stateid, 
+	* the confirmed flag is incorrecly set, or the generation 
+	* number is incorrect.  
+	* If there is no entry in the openfile table for this id, 
+	* we can't always return BAD_STATEID;
+	* this might be a retransmitted CLOSE which has arrived after 
+	* the openfile has been released.
+	*/
+	if (!(stp = find_stateid(stateid)))
+		goto no_nfs4_stateid;
+
+	status = nfserr_bad_stateid;
+
+	if ((flags & CHECK_FH) && nfs4_check_fh(current_fh, stp)) {
+		printk("NFSD: preprocess_seqid_op: fh-stateid mismatch!\n");
+		goto out;
+	}
+
+	*stpp = stp;
+	*sopp = sop = stp->st_stateowner;
+
+	/*
+	*  We now validate the seqid and stateid generation numbers.
+	*  For the moment, we ignore the possibility of 
+	*  generation number wraparound.
+	*/
+	if (seqid != sop->so_seqid + 1)
+		goto check_replay;
+
+	if (sop->so_confirmed) {
+		if (flags & CONFIRM) {
+			printk("NFSD: preprocess_seqid_op: expected unconfirmed stateowner!\n");
+			goto out;
+		}
+	}
+	else {
+		if (!(flags & CONFIRM)) {
+			printk("NFSD: preprocess_seqid_op: stateowner not confirmed yet!\n");
+			goto out;
+		}
+	}
+	if (stateid->si_generation > stp->st_stateid.si_generation) {
+		printk("NFSD: preprocess_seqid_op: future stateid?!\n");
+		goto out;
+	}
+
+	status = nfserr_old_stateid;
+	if (stateid->si_generation < stp->st_stateid.si_generation) {
+		printk("NFSD: preprocess_seqid_op: old stateid!\n");
+		goto out;
+	}
+	/* XXX renew the client lease here */
+	status = nfs_ok;
+
+out:
+	return status;
+
+no_nfs4_stateid:
+
+	/*
+	* We determine whether this is a bad stateid or a replay, 
+	* starting by trying to look up the stateowner.
+	* If stateowner is not found - stateid is bad.
+	*/
+	if (!(sop = find_stateowner_id(stateid->si_stateownerid))) {
+		printk("NFSD: preprocess_seqid_op: no stateowner or nfs4_stateid!\n");
+		status = nfserr_bad_stateid;
+		goto out;
+	}
+
+check_replay:
+	status = nfserr_bad_seqid;
+	if (seqid == sop->so_seqid) {
+		printk("NFSD: preprocess_seqid_op: retransmission?\n");
+		/* XXX will need to indicate replay to calling function here */
+	} else 
+		printk("NFSD: preprocess_seqid_op: bad seqid (expected %d, got %d\n", sop->so_seqid +1, seqid);
+
+	goto out;
+}
+
+int
+nfsd4_open_confirm(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfsd4_open_confirm *oc)
+{
+	int status;
+	struct nfs4_stateowner *sop;
+	struct nfs4_stateid *stp;
+
+	dprintk("NFSD: nfsd4_open_confirm on file %s\n",
+			current_fh->fh_dentry->d_name);
+	oc->oc_stateowner = NULL;
+	down(&client_sema); /* XXX need finer grained locking */
+
+	if ((status = nfs4_preprocess_seqid_op(current_fh, oc->oc_seqid,
+					&oc->oc_req_stateid,
+					CHECK_FH | CONFIRM,
+					&oc->oc_stateowner, &stp)))
+		goto out; 
+
+	sop = oc->oc_stateowner;
+	sop->so_confirmed = 1;
+	update_stateid(&stp->st_stateid);
+	memcpy(&oc->oc_resp_stateid, &stp->st_stateid, sizeof(stateid_t));
+	/* XXX renew the client lease here */
+	dprintk("NFSD: nfsd4_open_confirm: success, seqid=%d " 
+		"stateid=(%08x/%08x/%08x/%08x)\n", oc->oc_seqid,
+		         stp->st_stateid.si_boot,
+		         stp->st_stateid.si_stateownerid,
+		         stp->st_stateid.si_fileid,
+		         stp->st_stateid.si_generation);
+	status = nfs_ok;
+out:
+	up(&client_sema);
+	return status;
+}
 void 
 nfs4_state_init(void)
 {
@@ -1172,7 +1400,14 @@ nfs4_state_init(void)
 	}
 	for (i = 0; i < OWNER_HASH_SIZE; i++) {
 		INIT_LIST_HEAD(&ownerstr_hashtbl[i]);
+		INIT_LIST_HEAD(&ownerid_hashtbl[i]);
+	}
+	for (i = 0; i < OPENSTATEID_HASH_SIZE; i++) {
+		INIT_LIST_HEAD(&openstateid_hashtbl[i]);
 	}
+	memset(&zerostateid, 0, sizeof(stateid_t));
+	memset(&onestateid, ~0, sizeof(stateid_t));
+
 	INIT_LIST_HEAD(&client_lru);
 	init_MUTEX(&client_sema);
 	boot_time = get_seconds();

fs/nfsd/nfs4xdr.c

@@ -659,6 +659,19 @@ nfsd4_decode_open(struct nfsd4_compoundargs *argp, struct nfsd4_open *open)
 	DECODE_TAIL;
 }
 
+static int
+nfsd4_decode_open_confirm(struct nfsd4_compoundargs *argp, struct nfsd4_open_confirm *open_conf)
+{
+	DECODE_HEAD;
+		    
+	READ_BUF(4 + sizeof(stateid_t));
+	READ32(open_conf->oc_req_stateid.si_generation);
+	COPYMEM(&open_conf->oc_req_stateid.si_opaque, sizeof(stateid_opaque_t));
+	READ32(open_conf->oc_seqid);
+						        
+	DECODE_TAIL;
+}
+
 static int
 nfsd4_decode_putfh(struct nfsd4_compoundargs *argp, struct nfsd4_putfh *putfh)
 {
@@ -953,6 +966,9 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
 		case OP_OPEN:
 			op->status = nfsd4_decode_open(argp, &op->u.open);
 			break;
+		case OP_OPEN_CONFIRM:
+			op->status = nfsd4_decode_open_confirm(argp, &op->u.open_confirm);
+			break;
 		case OP_PUTFH:
 			op->status = nfsd4_decode_putfh(argp, &op->u.putfh);
 			break;
@@ -1057,6 +1073,21 @@ nfsd4_decode_compound(struct nfsd4_compoundargs *argp)
 } while (0)
 #define ADJUST_ARGS()		resp->p = p
 
+/*
+ * Routine for encoding the result of a
+ * "seqid-mutating" NFSv4 operation.  This is
+ * where seqids are incremented
+ */
+
+#define ENCODE_SEQID_OP_TAIL(stateowner)		\
+	BUG_ON(!stateowner);				\
+	if (seqid_mutating_err(nfserr) && stateowner) {	\
+		if (stateowner->so_confirmed)		\
+			stateowner->so_seqid++;		\
+	}						\
+	return nfserr;
+
+
 static u32 nfs4_ftypes[16] = {
         NF4BAD,  NF4FIFO, NF4CHR, NF4BAD,
         NF4DIR,  NF4BAD,  NF4BLK, NF4BAD,
@@ -1701,6 +1732,21 @@ nfsd4_encode_open(struct nfsd4_compoundres *resp, int nfserr, struct nfsd4_open
 	}
 }
 
+static int
+nfsd4_encode_open_confirm(struct nfsd4_compoundres *resp, int nfserr, struct nfsd4_open_confirm *oc)
+{
+	ENCODE_HEAD;
+				        
+	if (!nfserr) {
+		RESERVE_SPACE(sizeof(stateid_t));
+		WRITE32(oc->oc_resp_stateid.si_generation);
+		WRITEMEM(&oc->oc_resp_stateid.si_opaque, sizeof(stateid_opaque_t));
+		ADJUST_ARGS();
+	}
+
+	ENCODE_SEQID_OP_TAIL(oc->oc_stateowner);
+}
+
 static int
 nfsd4_encode_read(struct nfsd4_compoundres *resp, int nfserr, struct nfsd4_read *read)
 {
@@ -2013,6 +2059,9 @@ nfsd4_encode_operation(struct nfsd4_compoundres *resp, struct nfsd4_op *op)
 	case OP_OPEN:
 		nfsd4_encode_open(resp, op->status, &op->u.open);
 		break;
+	case OP_OPEN_CONFIRM:
+		nfsd4_encode_open_confirm(resp, op->status, &op->u.open_confirm);
+		break;
 	case OP_PUTFH:
 		break;
 	case OP_PUTROOTFH:

include/linux/nfsd/state.h

@@ -92,7 +92,6 @@ update_stateid(stateid_t *stateid)
 stateid->si_generation++;
 }
 
-
 /*
 * nfs4_stateowner can either be an open_owner, or (eventually) a lock_owner
 *
@@ -100,6 +99,7 @@ stateid->si_generation++;
 *              reverences when we release a stateowner.
 */
 struct nfs4_stateowner {
+	struct list_head        so_idhash;   /* hash by so_id */
 	struct list_head        so_strhash;   /* hash by op_name */
 	struct list_head        so_perclient; /* nfs4_client->cl_perclient */
 	struct list_head        so_peropenstate; /* list: nfs4_stateid */
@@ -136,6 +136,7 @@ struct nfs4_file {
 */
 
 struct nfs4_stateid {
+	struct list_head              st_hash; /* openstateid_hashtbl[]*/
 	struct list_head              st_perfile; /* file_hashtbl[]*/
 	struct list_head              st_peropenstate; /* nfs4_stateowner->so_peropenstate */
 	struct nfs4_stateowner      * st_stateowner;
@@ -147,4 +148,14 @@ struct nfs4_stateid {
 	unsigned int                  st_share_deny;
 };
 
+/* flags for preprocess_seqid_op() */
+#define CHECK_FH                0x00000001
+#define CONFIRM                 0x00000002
+
+#define seqid_mutating_err(err)                       \
+	(((err) != nfserr_stale_clientid) &&    \
+	((err) != nfserr_bad_seqid) &&          \
+	((err) != nfserr_stale_stateid) &&      \
+	((err) != nfserr_bad_stateid))
+
 #endif   /* NFSD4_STATE_H */

include/linux/nfsd/xdr4.h

@@ -147,6 +147,13 @@ struct nfsd4_open {
 #define op_iattr	u.iattr
 #define op_verf		u.verf
 
+struct nfsd4_open_confirm {
+	stateid_t	oc_req_stateid		/* request */;
+	u32		oc_seqid    		/* request */;
+	stateid_t	oc_resp_stateid		/* response */;
+	struct nfs4_stateowner * oc_stateowner;	/* response */
+};
+
 struct nfsd4_read {
 	stateid_t	rd_stateid;         /* request */
 	u64		rd_offset;          /* request */
@@ -252,6 +259,7 @@ struct nfsd4_op {
 		struct nfsd4_lookup		lookup;
 		struct nfsd4_verify		nverify;
 		struct nfsd4_open		open;
+		struct nfsd4_open_confirm	open_confirm;
 		struct nfsd4_putfh		putfh;
 		struct nfsd4_read		read;
 		struct nfsd4_readdir		readdir;
@@ -315,16 +323,23 @@ set_change_info(struct nfsd4_change_info *cinfo, struct svc_fh *fhp)
 }
 
 int nfs4svc_encode_voidres(struct svc_rqst *, u32 *, void *);
-int nfs4svc_decode_compoundargs(struct svc_rqst *, u32 *, struct nfsd4_compoundargs *);
-int nfs4svc_encode_compoundres(struct svc_rqst *, u32 *, struct nfsd4_compoundres *);
-
+int nfs4svc_decode_compoundargs(struct svc_rqst *, u32 *, 
+		struct nfsd4_compoundargs *);
+int nfs4svc_encode_compoundres(struct svc_rqst *, u32 *, 
+		struct nfsd4_compoundres *);
 void nfsd4_encode_operation(struct nfsd4_compoundres *, struct nfsd4_op *);
 int nfsd4_encode_fattr(struct svc_fh *fhp, struct svc_export *exp,
-		       struct dentry *dentry, u32 *buffer, int *countp, u32 *bmval);
-extern int nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_setclientid *setclid);
-extern int nfsd4_setclientid_confirm(struct svc_rqst *rqstp, struct nfsd4_setclientid_confirm *setclientid_confirm);
+		       struct dentry *dentry, u32 *buffer, int *countp, 
+		       u32 *bmval);
+extern int nfsd4_setclientid(struct svc_rqst *rqstp, 
+		struct nfsd4_setclientid *setclid);
+extern int nfsd4_setclientid_confirm(struct svc_rqst *rqstp, 
+		struct nfsd4_setclientid_confirm *setclientid_confirm);
 extern int nfsd4_process_open1(struct nfsd4_open *open);
-extern int nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nfsd4_open *open);
+extern int nfsd4_process_open2(struct svc_rqst *rqstp, 
+		struct svc_fh *current_fh, struct nfsd4_open *open);
+extern int nfsd4_open_confirm(struct svc_rqst *rqstp, 
+		struct svc_fh *current_fh, struct nfsd4_open_confirm *oc);
 #endif
 
 /*