NFSv4: Convert the lease renewal daemon from being

per-mountpoint to being per-server. Instead of running it on
top of rpciod, convert it to use keventd. This mean we can use
the struct nfs4_client semaphores for ordering purposes.

fs/nfs/inode.c

@@ -163,6 +163,8 @@ nfs_put_super(struct super_block *sb)
 		nfs_idmap_delete(server);
 #endif /* CONFIG_NFS_V4 */
 
+	nfs4_renewd_prepare_shutdown(server);
+
 	if (server->client != NULL)
 		rpc_shutdown_client(server->client);
 	if (server->client_sys != NULL)
@@ -1281,6 +1283,8 @@ static struct super_block *nfs_get_sb(struct file_system_type *fs_type,
 	if (!server)
 		return ERR_PTR(-ENOMEM);
 	memset(server, 0, sizeof(struct nfs_server));
+	/* Zero out the NFS state stuff */
+	init_nfsv4_state(server);
 
 	root = &server->fh;
 	memcpy(root, &data->root, sizeof(*root));
@@ -1444,13 +1448,15 @@ static int nfs4_fill_super(struct super_block *sb, struct nfs4_mount_data *data,
 		clp->cl_cred = rpcauth_lookupcred(clnt->cl_auth, 0);
 		memcpy(clp->cl_ipaddr, server->ip_addr, sizeof(clp->cl_ipaddr));
 	}
+	list_add_tail(&server->nfs4_siblings, &clp->cl_superblocks);
 	clnt = rpc_clone_client(clp->cl_rpcclient);
 	server->nfs4_state = clp;
 	up_write(&clp->cl_sem);
+	clp = NULL;
 
 	if (clnt == NULL) {
 		printk(KERN_WARNING "NFS: cannot create RPC client.\n");
-		goto out_fail;
+		goto out_remove_list;
 	}
 
 	clnt->cl_intr     = (server->flags & NFS4_MOUNT_INTR) ? 1 : 0;
@@ -1476,13 +1482,16 @@ static int nfs4_fill_super(struct super_block *sb, struct nfs4_mount_data *data,
 	err = nfs_sb_init(sb, authflavour);
 	if (err == 0)
 		return 0;
-	clp = NULL;
 	rpciod_down();
-	destroy_nfsv4_state(server);
 	if (server->idmap != NULL)
 		nfs_idmap_delete(server);
 out_shutdown:
 	rpc_shutdown_client(server->client);
+out_remove_list:
+	down_write(&server->nfs4_state->cl_sem);
+	list_del_init(&server->nfs4_siblings);
+	up_write(&server->nfs4_state->cl_sem);
+	destroy_nfsv4_state(server);
 out_fail:
 	if (clp)
 		nfs4_put_client(clp);
@@ -1542,6 +1551,8 @@ static struct super_block *nfs4_get_sb(struct file_system_type *fs_type,
 	if (!server)
 		return ERR_PTR(-ENOMEM);
 	memset(server, 0, sizeof(struct nfs_server));
+	/* Zero out the NFS state stuff */
+	init_nfsv4_state(server);
 
 	if (data->version != NFS4_MOUNT_VERSION) {
 		printk("nfs warning: mount version %s than kernel\n",

fs/nfs/nfs4proc.c

@@ -56,8 +56,6 @@ extern struct rpc_procinfo nfs4_procedures[];
 
 extern nfs4_stateid zero_stateid;
 
-static spinlock_t renew_lock = SPIN_LOCK_UNLOCKED;
-
 static void
 nfs4_setup_compound(struct nfs4_compound *cp, struct nfs4_op *ops,
 		    struct nfs_server *server, char *tag)
@@ -480,10 +478,11 @@ nfs4_setup_setclientid_confirm(struct nfs4_compound *cp)
 static void
 renew_lease(struct nfs_server *server, unsigned long timestamp)
 {
-	spin_lock(&renew_lock);
-	if (time_before(server->last_renewal,timestamp))
-		server->last_renewal = timestamp;
-	spin_unlock(&renew_lock);
+	struct nfs4_client *clp = server->nfs4_state;
+	spin_lock(&clp->cl_lock);
+	if (time_before(clp->cl_last_renewal,timestamp))
+		clp->cl_last_renewal = timestamp;
+	spin_unlock(&clp->cl_lock);
 }
 
 static inline void
@@ -748,6 +747,7 @@ nfs4_proc_get_root(struct nfs_server *server, struct nfs_fh *fhandle,
 	struct nfs_fsinfo	fsinfo;
 	unsigned char *		p;
 	struct qstr		q;
+	unsigned long		last_renewed;
 	int			status;
 
 	clp = server->nfs4_state;
@@ -773,6 +773,7 @@ nfs4_proc_get_root(struct nfs_server *server, struct nfs_fh *fhandle,
 	 */
 	nfs4_setup_compound(&compound, ops, server, "setclientid");
 	nfs4_setup_setclientid(&compound, 0, 0);
+	last_renewed = jiffies;
 	if ((status = nfs4_call_compound(&compound, NULL, 0)))
 		goto out_unlock;
 
@@ -786,20 +787,22 @@ nfs4_proc_get_root(struct nfs_server *server, struct nfs_fh *fhandle,
 	nfs4_setup_putrootfh(&compound);
 	nfs4_setup_getrootattr(&compound, fattr, &fsinfo);
 	nfs4_setup_getfh(&compound, fhandle);
+	last_renewed = jiffies;
 	if ((status = nfs4_call_compound(&compound, NULL, 0)))
 		goto out_unlock;
-	clp->cl_state = NFS4CLNT_OK;
 
-no_setclientid:
 	/*
 	 * Now that we have instantiated the clientid and determined
 	 * the lease time, we can initialize the renew daemon for this
 	 * server.
 	 * FIXME: we only need one renewd daemon per server.
 	 */
-	server->lease_time = fsinfo.lease_time * HZ;
-	if ((status = nfs4_init_renewd(server)))
-		goto out_unlock;
+	clp->cl_lease_time = fsinfo.lease_time * HZ;
+	clp->cl_last_renewal = last_renewed;
+	nfs4_schedule_state_renewal(clp);
+	clp->cl_state = NFS4CLNT_OK;
+
+no_setclientid:
 	up_write(&clp->cl_sem);
 	
 	/*
@@ -1642,22 +1645,24 @@ nfs4_proc_commit_setup(struct nfs_write_data *data, u64 start, u32 len, int how)
 static void
 renew_done(struct rpc_task *task)
 {
-	struct nfs_server *server = (struct nfs_server *)task->tk_msg.rpc_resp;
+	struct nfs4_client *clp = (struct nfs4_client *)task->tk_msg.rpc_argp;
 	unsigned long timestamp = (unsigned long)task->tk_calldata;
-	renew_lease(server, timestamp);
+	spin_lock(&clp->cl_lock);
+	if (time_before(clp->cl_last_renewal,timestamp))
+		clp->cl_last_renewal = timestamp;
+	spin_unlock(&clp->cl_lock);
 }
 
 int
-nfs4_proc_async_renew(struct nfs_server *server, struct rpc_cred *cred)
+nfs4_proc_async_renew(struct nfs4_client *clp)
 {
 	struct rpc_message msg = {
 		.rpc_proc	= &nfs4_procedures[NFSPROC4_CLNT_RENEW],
-		.rpc_argp	= server->nfs4_state,
-		.rpc_resp	= server,
-		.rpc_cred	= cred,
+		.rpc_argp	= clp,
+		.rpc_cred	= clp->cl_cred,
 	};
 
-	return rpc_call_async(server->client, &msg, 0, renew_done, (void *)jiffies);
+	return rpc_call_async(clp->cl_rpcclient, &msg, 0, renew_done, (void *)jiffies);
 }
 
 /*

fs/nfs/nfs4renewd.c

@@ -54,53 +54,91 @@
 #include <linux/nfs4.h>
 #include <linux/nfs_fs.h>
 
-static RPC_WAITQ(nfs4_renewd_queue, "nfs4_renewd_queue");
+#define NFSDBG_FACILITY	NFSDBG_PROC
 
-static void
-renewd(struct rpc_task *task)
+void
+nfs4_renew_state(void *data)
 {
-	struct nfs_server *server = (struct nfs_server *)task->tk_calldata;
-	unsigned long lease = server->lease_time;
-	unsigned long last = server->last_renewal;
-	unsigned long timeout;
+	struct nfs4_client *clp = (struct nfs4_client *)data;
+	long lease, timeout;
+	unsigned long last, now;
 
-	if (!server->nfs4_state)
-		timeout = (2 * lease) / 3;
-	else if (jiffies < last + lease/3)
-		timeout = (2 * lease) / 3 + last - jiffies;
-	else {
+	down_read(&clp->cl_sem);
+	dprintk("%s: start\n", __FUNCTION__);
+	/* Are there any active superblocks? */
+	if (list_empty(&clp->cl_superblocks))
+		goto out; 
+	spin_lock(&clp->cl_lock);
+	lease = clp->cl_lease_time;
+	last = clp->cl_last_renewal;
+	now = jiffies;
+	timeout = (2 * lease) / 3 + (long)last - (long)now;
+	/* Are we close to a lease timeout? */
+	if (time_after(now, last + lease/3)) {
+		spin_unlock(&clp->cl_lock);
 		/* Queue an asynchronous RENEW. */
-		nfs4_proc_async_renew(server, NULL);
+		nfs4_proc_async_renew(clp);
 		timeout = (2 * lease) / 3;
-	}
-
+		spin_lock(&clp->cl_lock);
+	} else
+		dprintk("%s: failed to call renewd. Reason: lease not expired \n",
+				__FUNCTION__);
 	if (timeout < 5 * HZ)    /* safeguard */
 		timeout = 5 * HZ;
-	task->tk_timeout = timeout;
-	task->tk_action = renewd;
-	task->tk_exit = NULL;
-	rpc_sleep_on(&nfs4_renewd_queue, task, NULL, NULL);
-	return;
+	dprintk("%s: requeueing work. Lease period = %ld\n",
+			__FUNCTION__, (timeout + HZ - 1) / HZ);
+	cancel_delayed_work(&clp->cl_renewd);
+	schedule_delayed_work(&clp->cl_renewd, timeout);
+	spin_unlock(&clp->cl_lock);
+out:
+	up_read(&clp->cl_sem);
+	dprintk("%s: done\n", __FUNCTION__);
 }
 
-int
-nfs4_init_renewd(struct nfs_server *server)
+/* Must be called with clp->cl_sem locked for writes */
+void
+nfs4_schedule_state_renewal(struct nfs4_client *clp)
 {
-	struct rpc_task *task;
-	int status;
+	long timeout;
 
-	lock_kernel();
-	status = -ENOMEM;
-	task = rpc_new_task(server->client, NULL, RPC_TASK_ASYNC);
-	if (!task)
-		goto out;
-	task->tk_calldata = server;
-	task->tk_action = renewd;
-	status = rpc_execute(task);
+	spin_lock(&clp->cl_lock);
+	timeout = (2 * clp->cl_lease_time) / 3 + (long)clp->cl_last_renewal
+		- (long)jiffies;
+	if (timeout < 5 * HZ)
+		timeout = 5 * HZ;
+	dprintk("%s: requeueing work. Lease period = %ld\n",
+			__FUNCTION__, (timeout + HZ - 1) / HZ);
+	cancel_delayed_work(&clp->cl_renewd);
+	schedule_delayed_work(&clp->cl_renewd, timeout);
+	spin_unlock(&clp->cl_lock);
+}
 
-out:
-	unlock_kernel();
-	return status;
+void
+nfs4_renewd_prepare_shutdown(struct nfs_server *server)
+{
+	struct nfs4_client *clp = server->nfs4_state;
+
+	if (!clp)
+		return;
+	flush_scheduled_work();
+	down_write(&clp->cl_sem);
+	if (!list_empty(&server->nfs4_siblings))
+		list_del_init(&server->nfs4_siblings);
+	up_write(&clp->cl_sem);
+}
+
+/* Must be called with clp->cl_sem locked for writes */
+void
+nfs4_kill_renewd(struct nfs4_client *clp)
+{
+	down_read(&clp->cl_sem);
+	if (!list_empty(&clp->cl_superblocks)) {
+		up_read(&clp->cl_sem);
+		return;
+	}
+	cancel_delayed_work(&clp->cl_renewd);
+	up_read(&clp->cl_sem);
+	flush_scheduled_work();
 }
 
 /*

fs/nfs/nfs4state.c

@@ -41,6 +41,7 @@
 #include <linux/config.h>
 #include <linux/slab.h>
 #include <linux/nfs_fs.h>
+#include <linux/workqueue.h>
 
 #define OPENOWNER_POOL_SIZE	8
 
@@ -55,6 +56,28 @@ nfs4_stateid one_stateid =
 
 static LIST_HEAD(nfs4_clientid_list);
 
+extern void nfs4_renew_state(void *);
+
+void
+init_nfsv4_state(struct nfs_server *server)
+{
+	server->nfs4_state = NULL;
+	INIT_LIST_HEAD(&server->nfs4_siblings);
+}
+
+void
+destroy_nfsv4_state(struct nfs_server *server)
+{
+	if (server->mnt_path) {
+		kfree(server->mnt_path);
+		server->mnt_path = NULL;
+	}
+	if (server->nfs4_state) {
+		nfs4_put_client(server->nfs4_state);
+		server->nfs4_state = NULL;
+	}
+}
+
 /*
  * nfs4_get_client(): returns an empty client structure
  * nfs4_put_client(): drops reference to client structure
@@ -75,6 +98,8 @@ nfs4_alloc_client(struct in_addr *addr)
 		INIT_LIST_HEAD(&clp->cl_unused);
 		spin_lock_init(&clp->cl_lock);
 		atomic_set(&clp->cl_count, 1);
+		INIT_WORK(&clp->cl_renewd, nfs4_renew_state, clp);
+		INIT_LIST_HEAD(&clp->cl_superblocks);
 		clp->cl_state = NFS4CLNT_NEW;
 	}
 	return clp;
@@ -130,6 +155,7 @@ nfs4_put_client(struct nfs4_client *clp)
 		return;
 	list_del(&clp->cl_servers);
 	spin_unlock(&state_spinlock);
+	nfs4_kill_renewd(clp);
 	nfs4_free_client(clp);
 }
 

include/linux/nfs_fs.h

@@ -28,6 +28,7 @@
 #include <linux/nfs3.h>
 #include <linux/nfs4.h>
 #include <linux/nfs_xdr.h>
+#include <linux/workqueue.h>
 
 /*
  * Enable debugging support for nfs client.
@@ -493,6 +494,12 @@ struct nfs4_client {
 	struct rpc_clnt *	cl_rpcclient;
 	struct rpc_cred *	cl_cred;
 
+	struct list_head	cl_superblocks;	/* List of nfs_server structs */
+
+	unsigned long		cl_lease_time;
+	unsigned long		cl_last_renewal;
+	struct work_struct	cl_renewd;
+
 	/* Our own IP address, as a null-terminated string.
 	 * This is used to generate the clientid, and the callback address.
 	 */
@@ -545,13 +552,17 @@ struct nfs4_state {
 
 
 /* nfs4proc.c */
-extern int nfs4_proc_async_renew(struct nfs_server *server, struct rpc_cred *);
+extern int nfs4_proc_async_renew(struct nfs4_client *);
 extern int nfs4_do_close(struct inode *, struct nfs4_state *);
 
 /* nfs4renewd.c */
-extern int nfs4_init_renewd(struct nfs_server *server);
+extern void nfs4_schedule_state_renewal(struct nfs4_client *);
+extern void nfs4_renewd_prepare_shutdown(struct nfs_server *);
+extern void nfs4_kill_renewd(struct nfs4_client *);
 
 /* nfs4state.c */
+extern void init_nfsv4_state(struct nfs_server *);
+extern void destroy_nfsv4_state(struct nfs_server *);
 extern struct nfs4_client *nfs4_get_client(struct in_addr *);
 extern void nfs4_put_client(struct nfs4_client *clp);
 extern struct nfs4_state_owner * nfs4_get_state_owner(struct nfs_server *, struct rpc_cred *);
@@ -560,29 +571,13 @@ extern struct nfs4_state * nfs4_get_open_state(struct inode *, struct nfs4_state
 extern void nfs4_put_open_state(struct nfs4_state *);
 extern void nfs4_increment_seqid(u32 status, struct nfs4_state_owner *sp);
 
-
-
-
-
-
 struct nfs4_mount_data;
-static inline void
-destroy_nfsv4_state(struct nfs_server *server)
-{
-	if (server->mnt_path) {
-		kfree(server->mnt_path);
-		server->mnt_path = NULL;
-	}
-	if (server->nfs4_state) {
-		nfs4_put_client(server->nfs4_state);
-		server->nfs4_state = NULL;
-	}
-}
 #else
-#define create_nfsv4_state(server, data)  0
+#define init_nfsv4_state(server)  do { } while (0)
 #define destroy_nfsv4_state(server)       do { } while (0)
 #define nfs4_put_state_owner(inode, owner) do { } while (0)
 #define nfs4_put_open_state(state) do { } while (0)
+#define nfs4_renewd_prepare_shutdown(server) do { } while (0)
 #endif
 
 #endif /* __KERNEL__ */

include/linux/nfs_fs_sb.h

@@ -35,8 +35,9 @@ struct nfs_server {
 	char			ip_addr[16];
 	char *			mnt_path;
 	struct nfs4_client *	nfs4_state;	/* all NFSv4 state starts here */
-	unsigned long		lease_time;	/* in jiffies */
-	unsigned long		last_renewal;	/* in jiffies */
+	struct list_head	nfs4_siblings;	/* List of other nfs_server structs
+						 * that share the same clientid
+						 */
 	void                   *idmap;
 #endif
 };