[PATCH] reduced overheads in fget/fput

From: Dipankar Sarma <dipankar@in.ibm.com>


fget() shows up on profiles, especially on SMP.  Dipankar's patch
special-cases the situation wherein there are no sharers of current->files.

In this situation we know that no other process can close this file, so it
is not necessary to increment the file's refcount.

It's ugly as sin, but makes a substantial difference.

The test is

	dd if=/dev/zero of=foo bs=1 count=1M

On 4CPU P3 xeon with 1MB L2 cache and 512MB ram:

	kernel           sys time     std-dev
	------------     --------     -------

	UP - vanilla     2.104        0.028
	UP - file        1.867        0.019

	SMP - vanilla    2.976        0.023
	SMP - file       2.719        0.026

fs/file_table.c

@@ -141,7 +141,7 @@ void close_private_file(struct file *file)
 	security_file_free(file);
 }
 
-void fput(struct file * file)
+void fput(struct file *file)
 {
 	if (atomic_dec_and_test(&file->f_count))
 		__fput(file);
@@ -190,6 +190,34 @@ struct file *fget(unsigned int fd)
 	return file;
 }
 
+/*
+ * Lightweight file lookup - no refcnt increment if fd table isn't shared. 
+ * You can use this only if it is guranteed that the current task already 
+ * holds a refcnt to that file. That check has to be done at fget() only
+ * and a flag is returned to be passed to the corresponding fput_light().
+ * There must not be a cloning between an fget_light/fput_light pair.
+ */
+struct file *fget_light(unsigned int fd, int *fput_needed)
+{
+	struct file *file;
+	struct files_struct *files = current->files;
+
+	*fput_needed = 0;
+	if (likely((atomic_read(&files->count) == 1))) {
+		file = fcheck(fd);
+	} else {
+		spin_lock(&files->file_lock);
+		file = fcheck(fd);
+		if (file) {
+			get_file(file);
+			*fput_needed = 1;
+		}
+		spin_unlock(&files->file_lock);
+	}
+	return file;
+}
+
+
 void put_filp(struct file *file)
 {
 	if (atomic_dec_and_test(&file->f_count)) {

fs/read_write.c

@@ -115,9 +115,10 @@ asmlinkage off_t sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
 {
 	off_t retval;
 	struct file * file;
+	int fput_needed;
 
 	retval = -EBADF;
-	file = fget(fd);
+	file = fget_light(fd, &fput_needed);
 	if (!file)
 		goto bad;
 
@@ -128,7 +129,7 @@ asmlinkage off_t sys_lseek(unsigned int fd, off_t offset, unsigned int origin)
 		if (res != (loff_t)retval)
 			retval = -EOVERFLOW;	/* LFS: should only happen on 32 bit platforms */
 	}
-	fput(file);
+	fput_light(file, fput_needed);
 bad:
 	return retval;
 }
@@ -141,9 +142,10 @@ asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
 	int retval;
 	struct file * file;
 	loff_t offset;
+	int fput_needed;
 
 	retval = -EBADF;
-	file = fget(fd);
+	file = fget_light(fd, &fput_needed);
 	if (!file)
 		goto bad;
 
@@ -161,7 +163,7 @@ asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high,
 			retval = 0;
 	}
 out_putf:
-	fput(file);
+	fput_light(file, fput_needed);
 bad:
 	return retval;
 }
@@ -251,11 +253,12 @@ asmlinkage ssize_t sys_read(unsigned int fd, char __user * buf, size_t count)
 {
 	struct file *file;
 	ssize_t ret = -EBADF;
+	int fput_needed;
 
-	file = fget(fd);
+	file = fget_light(fd, &fput_needed);
 	if (file) {
 		ret = vfs_read(file, buf, count, &file->f_pos);
-		fput(file);
+		fput_light(file, fput_needed);
 	}
 
 	return ret;
@@ -265,11 +268,12 @@ asmlinkage ssize_t sys_write(unsigned int fd, const char __user * buf, size_t co
 {
 	struct file *file;
 	ssize_t ret = -EBADF;
+	int fput_needed;
 
-	file = fget(fd);
+	file = fget_light(fd, &fput_needed);
 	if (file) {
 		ret = vfs_write(file, buf, count, &file->f_pos);
-		fput(file);
+		fput_light(file, fput_needed);
 	}
 
 	return ret;
@@ -280,14 +284,15 @@ asmlinkage ssize_t sys_pread64(unsigned int fd, char __user *buf,
 {
 	struct file *file;
 	ssize_t ret = -EBADF;
+	int fput_needed;
 
 	if (pos < 0)
 		return -EINVAL;
 
-	file = fget(fd);
+	file = fget_light(fd, &fput_needed);
 	if (file) {
 		ret = vfs_read(file, buf, count, &pos);
-		fput(file);
+		fput_light(file, fput_needed);
 	}
 
 	return ret;
@@ -298,14 +303,15 @@ asmlinkage ssize_t sys_pwrite64(unsigned int fd, const char __user *buf,
 {
 	struct file *file;
 	ssize_t ret = -EBADF;
+	int fput_needed;
 
 	if (pos < 0)
 		return -EINVAL;
 
-	file = fget(fd);
+	file = fget_light(fd, &fput_needed);
 	if (file) {
 		ret = vfs_write(file, buf, count, &pos);
-		fput(file);
+		fput_light(file, fput_needed);
 	}
 
 	return ret;
@@ -479,11 +485,12 @@ sys_readv(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
 {
 	struct file *file;
 	ssize_t ret = -EBADF;
+	int fput_needed;
 
-	file = fget(fd);
+	file = fget_light(fd, &fput_needed);
 	if (file) {
 		ret = vfs_readv(file, vec, vlen, &file->f_pos);
-		fput(file);
+		fput_light(file, fput_needed);
 	}
 
 	return ret;
@@ -494,11 +501,12 @@ sys_writev(unsigned long fd, const struct iovec __user *vec, unsigned long vlen)
 {
 	struct file *file;
 	ssize_t ret = -EBADF;
+	int fput_needed;
 
-	file = fget(fd);
+	file = fget_light(fd, &fput_needed);
 	if (file) {
 		ret = vfs_writev(file, vec, vlen, &file->f_pos);
-		fput(file);
+		fput_light(file, fput_needed);
 	}
 
 	return ret;
@@ -511,12 +519,13 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 	struct inode * in_inode, * out_inode;
 	loff_t pos;
 	ssize_t retval;
+	int fput_needed_in, fput_needed_out;
 
 	/*
 	 * Get input file, and verify that it is ok..
 	 */
 	retval = -EBADF;
-	in_file = fget(in_fd);
+	in_file = fget_light(in_fd, &fput_needed_in);
 	if (!in_file)
 		goto out;
 	if (!(in_file->f_mode & FMODE_READ))
@@ -539,7 +548,7 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 	 * Get output file, and verify that it is ok..
 	 */
 	retval = -EBADF;
-	out_file = fget(out_fd);
+	out_file = fget_light(out_fd, &fput_needed_out);
 	if (!out_file)
 		goto fput_in;
 	if (!(out_file->f_mode & FMODE_WRITE))
@@ -579,9 +588,9 @@ static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos,
 		retval = -EOVERFLOW;
 
 fput_out:
-	fput(out_file);
+	fput_light(out_file, fput_needed_out);
 fput_in:
-	fput(in_file);
+	fput_light(in_file, fput_needed_in);
 out:
 	return retval;
 }

include/linux/file.h

@@ -35,7 +35,15 @@ struct files_struct {
 
 extern void FASTCALL(__fput(struct file *));
 extern void FASTCALL(fput(struct file *));
+
+static inline void fput_light(struct file *file, int fput_needed)
+{
+	if (unlikely(fput_needed))
+		fput(file);
+}
+
 extern struct file * FASTCALL(fget(unsigned int fd));
+extern struct file * FASTCALL(fget_light(unsigned int fd, int *fput_needed));
 extern void FASTCALL(set_close_on_exec(unsigned int fd, int flag));
 extern void put_filp(struct file *);
 extern int get_unused_fd(void);