[PATCH] PATCH 12/16: NFSD: TCP: Close idle TCP connections

Close idle rpc/tcp sockets

We split the list of sv_allsocks into two, one
of permanent sockets (udp, tcp listener) and one
of temporary sockets (tcp data).

Whenever we complete a successful receive on a temp socket,
it gets pushed to the end of the list.

Whenever a thread wants to do something, it first checks if
the oldest temp socket has not has a receive for 6 mintutes
(should possibly be configurable).  It so, we simulate
a close.

Finally we make sure that threads wake up every few minutes
so that if the server is completely idle, all temp
sockets will get closed.

fs/nfsd/nfssvc.c

@@ -195,7 +195,7 @@ nfsd(struct svc_rqst *rqstp)
 		 * recvfrom routine.
 		 */
 		while ((err = svc_recv(serv, rqstp,
-				       MAX_SCHEDULE_TIMEOUT)) == -EAGAIN)
+				       5*60*HZ)) == -EAGAIN)
 		    ;
 		if (err < 0)
 			break;

include/linux/sunrpc/svc.h

@@ -36,7 +36,8 @@ struct svc_serv {
 	unsigned int		sv_bufsz;	/* datagram buffer size */
 	unsigned int		sv_xdrsize;	/* XDR buffer size */
 
-	struct list_head	sv_allsocks;	/* all sockets */
+	struct list_head	sv_permsocks;	/* all permanent sockets */
+	struct list_head	sv_tempsocks;	/* all temporary sockets */
 
 	char *			sv_name;	/* service name */
 };

include/linux/sunrpc/svcsock.h

@@ -41,6 +41,7 @@ struct svc_sock {
 	/* private TCP part */
 	int			sk_reclen;	/* length of record */
 	int			sk_tcplen;	/* current read length */
+	time_t			sk_lastrecv;	/* time of last received request */
 };
 
 /*

net/sunrpc/svc.c

@@ -42,7 +42,8 @@ svc_create(struct svc_program *prog, unsigned int bufsize, unsigned int xdrsize)
 	serv->sv_xdrsize   = xdrsize;
 	INIT_LIST_HEAD(&serv->sv_threads);
 	INIT_LIST_HEAD(&serv->sv_sockets);
-	INIT_LIST_HEAD(&serv->sv_allsocks);
+	INIT_LIST_HEAD(&serv->sv_tempsocks);
+	INIT_LIST_HEAD(&serv->sv_permsocks);
 	spin_lock_init(&serv->sv_lock);
 
 	serv->sv_name      = prog->pg_name;
@@ -71,8 +72,14 @@ svc_destroy(struct svc_serv *serv)
 	} else
 		printk("svc_destroy: no threads for serv=%p!\n", serv);
 
-	while (!list_empty(&serv->sv_allsocks)) {
-		svsk = list_entry(serv->sv_allsocks.next,
+	while (!list_empty(&serv->sv_tempsocks)) {
+		svsk = list_entry(serv->sv_tempsocks.next,
+				  struct svc_sock,
+				  sk_list);
+		svc_delete_socket(svsk);
+	}
+	while (!list_empty(&serv->sv_permsocks)) {
+		svsk = list_entry(serv->sv_permsocks.next,
 				  struct svc_sock,
 				  sk_list);
 		svc_delete_socket(svsk);

net/sunrpc/svcsock.c

@@ -585,7 +585,6 @@ svc_tcp_accept(struct svc_sock *svsk)
 	 * installed the data_ready callback. 
 	 */
 	set_bit(SK_DATA, &newsvsk->sk_flags);
-	set_bit(SK_TEMP, &newsvsk->sk_flags);
 	svc_sock_enqueue(newsvsk);
 
 	if (serv->sv_stats)
@@ -781,7 +780,7 @@ svc_tcp_init(struct svc_sock *svsk)
 int
 svc_recv(struct svc_serv *serv, struct svc_rqst *rqstp, long timeout)
 {
-	struct svc_sock		*svsk;
+	struct svc_sock		*svsk =NULL;
 	int			len;
 	DECLARE_WAITQUEUE(wait, current);
 
@@ -805,7 +804,24 @@ svc_recv(struct svc_serv *serv, struct svc_rqst *rqstp, long timeout)
 		return -EINTR;
 
 	spin_lock_bh(&serv->sv_lock);
-	if ((svsk = svc_sock_dequeue(serv)) != NULL) {
+	if (!list_empty(&serv->sv_tempsocks)) {
+		svsk = list_entry(serv->sv_tempsocks.next,
+				  struct svc_sock, sk_list);
+		/* apparently the "standard" is that clients close
+		 * idle connections after 5 minutes, servers after
+		 * 6 minutes
+		 *   http://www.connectathon.org/talks96/nfstcp.pdf 
+		 */
+		if (CURRENT_TIME - svsk->sk_lastrecv < 6*60
+		    || test_bit(SK_BUSY, &svsk->sk_flags))
+			svsk = NULL;
+	}
+	if (svsk) {
+		set_bit(SK_BUSY, &svsk->sk_flags);
+		set_bit(SK_CLOSE, &svsk->sk_flags);
+		rqstp->rq_sock = svsk;
+		svsk->sk_inuse++;
+	} else if ((svsk = svc_sock_dequeue(serv)) != NULL) {
 		rqstp->rq_sock = svsk;
 		svsk->sk_inuse++;
 	} else {
@@ -844,6 +860,14 @@ svc_recv(struct svc_serv *serv, struct svc_rqst *rqstp, long timeout)
 		svc_sock_release(rqstp);
 		return -EAGAIN;
 	}
+	svsk->sk_lastrecv = CURRENT_TIME;
+	if (test_bit(SK_TEMP, &svsk->sk_flags)) {
+		/* push active sockets to end of list */
+		spin_lock_bh(&serv->sv_lock);
+		list_del(&svsk->sk_list);
+		list_add_tail(&svsk->sk_list, &serv->sv_tempsocks);
+		spin_unlock_bh(&serv->sv_lock);
+	}
 
 	rqstp->rq_secure  = ntohs(rqstp->rq_addr.sin_port) < 1024;
 	rqstp->rq_userset = 0;
@@ -921,6 +945,7 @@ svc_setup_socket(struct svc_serv *serv, struct socket *sock,
 	svsk->sk_ostate = inet->state_change;
 	svsk->sk_odata = inet->data_ready;
 	svsk->sk_server = serv;
+	svsk->sk_lastrecv = CURRENT_TIME;
 
 	/* Initialize the socket */
 	if (sock->type == SOCK_DGRAM)
@@ -940,8 +965,15 @@ if (svsk->sk_sk == NULL)
 		return NULL;
 	}
 
+
 	spin_lock_bh(&serv->sv_lock);
-	list_add(&svsk->sk_list, &serv->sv_allsocks);
+	if (!pmap_register) {
+		set_bit(SK_TEMP, &svsk->sk_flags);
+		list_add(&svsk->sk_list, &serv->sv_tempsocks);
+	} else {
+		clear_bit(SK_TEMP, &svsk->sk_flags);
+		list_add(&svsk->sk_list, &serv->sv_permsocks);
+	}
 	spin_unlock_bh(&serv->sv_lock);
 
 	dprintk("svc: svc_setup_socket created %p (inet %p)\n",