Commit 65178db4 authored by Bryan Schumaker's avatar Bryan Schumaker Committed by J. Bruce Fields

NFSD: Added fault injection

Fault injection on the NFS server makes it easier to test the client's
state manager and recovery threads.  Simulating errors on the server is
easier than finding the right conditions that cause them naturally.

This patch uses debugfs to add a simple framework for fault injection to
the server.  This framework is a config option, and can be enabled
through CONFIG_NFSD_FAULT_INJECTION.  Assuming you have debugfs mounted
to /sys/debug, a set of files will be created in /sys/debug/nfsd/.
Writing to any of these files will cause the corresponding action and
write a log entry to dmesg.
Signed-off-by: default avatarBryan Schumaker <bjschuma@netapp.com>
Signed-off-by: default avatarJ. Bruce Fields <bfields@redhat.com>
parent 64a284d0
...@@ -80,3 +80,13 @@ config NFSD_V4 ...@@ -80,3 +80,13 @@ config NFSD_V4
available from http://linux-nfs.org/. available from http://linux-nfs.org/.
If unsure, say N. If unsure, say N.
config NFSD_FAULT_INJECTION
bool "NFS server manual fault injection"
depends on NFSD_V4 && DEBUG_KERNEL
help
This option enables support for manually injecting faults
into the NFS server. This is intended to be used for
testing error recovery on the NFS client.
If unsure, say N.
...@@ -6,6 +6,7 @@ obj-$(CONFIG_NFSD) += nfsd.o ...@@ -6,6 +6,7 @@ obj-$(CONFIG_NFSD) += nfsd.o
nfsd-y := nfssvc.o nfsctl.o nfsproc.o nfsfh.o vfs.o \ nfsd-y := nfssvc.o nfsctl.o nfsproc.o nfsfh.o vfs.o \
export.o auth.o lockd.o nfscache.o nfsxdr.o stats.o export.o auth.o lockd.o nfscache.o nfsxdr.o stats.o
nfsd-$(CONFIG_NFSD_FAULT_INJECTION) += fault_inject.o
nfsd-$(CONFIG_NFSD_V2_ACL) += nfs2acl.o nfsd-$(CONFIG_NFSD_V2_ACL) += nfs2acl.o
nfsd-$(CONFIG_NFSD_V3) += nfs3proc.o nfs3xdr.o nfsd-$(CONFIG_NFSD_V3) += nfs3proc.o nfs3xdr.o
nfsd-$(CONFIG_NFSD_V3_ACL) += nfs3acl.o nfsd-$(CONFIG_NFSD_V3_ACL) += nfs3acl.o
......
/*
* Copyright (c) 2011 Bryan Schumaker <bjschuma@netapp.com>
*
* Uses debugfs to create fault injection points for client testing
*/
#include <linux/types.h>
#include <linux/fs.h>
#include <linux/debugfs.h>
#include <linux/module.h>
#include "state.h"
#include "fault_inject.h"
struct nfsd_fault_inject_op {
char *file;
void (*func)(u64);
};
static struct nfsd_fault_inject_op inject_ops[] = {
{
.file = "forget_clients",
.func = nfsd_forget_clients,
},
{
.file = "forget_locks",
.func = nfsd_forget_locks,
},
{
.file = "forget_openowners",
.func = nfsd_forget_openowners,
},
{
.file = "forget_delegations",
.func = nfsd_forget_delegations,
},
{
.file = "recall_delegations",
.func = nfsd_recall_delegations,
},
};
static long int NUM_INJECT_OPS = sizeof(inject_ops) / sizeof(struct nfsd_fault_inject_op);
static struct dentry *debug_dir;
static int nfsd_inject_set(void *op_ptr, u64 val)
{
struct nfsd_fault_inject_op *op = op_ptr;
if (val == 0)
printk(KERN_INFO "NFSD Fault Injection: %s (all)", op->file);
else
printk(KERN_INFO "NFSD Fault Injection: %s (n = %llu)", op->file, val);
op->func(val);
return 0;
}
static int nfsd_inject_get(void *data, u64 *val)
{
return 0;
}
DEFINE_SIMPLE_ATTRIBUTE(fops_nfsd, nfsd_inject_get, nfsd_inject_set, "%llu\n");
void nfsd_fault_inject_cleanup(void)
{
debugfs_remove_recursive(debug_dir);
}
int nfsd_fault_inject_init(void)
{
unsigned int i;
struct nfsd_fault_inject_op *op;
mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
debug_dir = debugfs_create_dir("nfsd", NULL);
if (!debug_dir)
goto fail;
for (i = 0; i < NUM_INJECT_OPS; i++) {
op = &inject_ops[i];
if (!debugfs_create_file(op->file, mode, debug_dir, op, &fops_nfsd))
goto fail;
}
return 0;
fail:
nfsd_fault_inject_cleanup();
return -ENOMEM;
}
/*
* Copyright (c) 2011 Bryan Schumaker <bjschuma@netapp.com>
*
* Function definitions for fault injection
*/
#ifndef LINUX_NFSD_FAULT_INJECT_H
#define LINUX_NFSD_FAULT_INJECT_H
#ifdef CONFIG_NFSD_FAULT_INJECTION
int nfsd_fault_inject_init(void);
void nfsd_fault_inject_cleanup(void);
void nfsd_forget_clients(u64);
void nfsd_forget_locks(u64);
void nfsd_forget_openowners(u64);
void nfsd_forget_delegations(u64);
void nfsd_recall_delegations(u64);
#else /* CONFIG_NFSD_FAULT_INJECTION */
static inline int nfsd_fault_inject_init(void) { return 0; }
static inline void nfsd_fault_inject_cleanup(void) {}
static inline void nfsd_forget_clients(u64 num) {}
static inline void nfsd_forget_locks(u64 num) {}
static inline void nfsd_forget_openowners(u64 num) {}
static inline void nfsd_forget_delegations(u64 num) {}
static inline void nfsd_recall_delegations(u64 num) {}
#endif /* CONFIG_NFSD_FAULT_INJECTION */
#endif /* LINUX_NFSD_FAULT_INJECT_H */
...@@ -4429,6 +4429,121 @@ nfs4_check_open_reclaim(clientid_t *clid) ...@@ -4429,6 +4429,121 @@ nfs4_check_open_reclaim(clientid_t *clid)
return nfs4_find_reclaim_client(clid) ? nfs_ok : nfserr_reclaim_bad; return nfs4_find_reclaim_client(clid) ? nfs_ok : nfserr_reclaim_bad;
} }
#ifdef CONFIG_NFSD_FAULT_INJECTION
void nfsd_forget_clients(u64 num)
{
struct nfs4_client *clp, *next;
int count = 0;
nfs4_lock_state();
list_for_each_entry_safe(clp, next, &client_lru, cl_lru) {
nfsd4_remove_clid_dir(clp);
expire_client(clp);
if (++count == num)
break;
}
nfs4_unlock_state();
printk(KERN_INFO "NFSD: Forgot %d clients", count);
}
static void release_lockowner_sop(struct nfs4_stateowner *sop)
{
release_lockowner(lockowner(sop));
}
static void release_openowner_sop(struct nfs4_stateowner *sop)
{
release_openowner(openowner(sop));
}
static int nfsd_release_n_owners(u64 num,
struct list_head hashtbl[],
unsigned int hashtbl_size,
void (*release_sop)(struct nfs4_stateowner *))
{
int i, count = 0;
struct nfs4_stateowner *sop, *next;
for (i = 0; i < hashtbl_size; i++) {
list_for_each_entry_safe(sop, next, &hashtbl[i], so_strhash) {
release_sop(sop);
if (++count == num)
return count;
}
}
return count;
}
void nfsd_forget_locks(u64 num)
{
int count;
nfs4_lock_state();
count = nfsd_release_n_owners(num, lock_ownerstr_hashtbl,
LOCK_HASH_SIZE, release_lockowner_sop);
nfs4_unlock_state();
printk(KERN_INFO "NFSD: Forgot %d locks", count);
}
void nfsd_forget_openowners(u64 num)
{
int count;
nfs4_lock_state();
count = nfsd_release_n_owners(num, open_ownerstr_hashtbl,
OPEN_OWNER_HASH_SIZE, release_openowner_sop);
nfs4_unlock_state();
printk(KERN_INFO "NFSD: Forgot %d open owners", count);
}
int nfsd_process_n_delegations(u64 num, void (*deleg_func)(struct nfs4_delegation *))
{
int i, count = 0;
struct nfs4_file *fp;
struct nfs4_delegation *dp, *next;
for (i = 0; i < FILE_HASH_SIZE; i++) {
list_for_each_entry(fp, &file_hashtbl[i], fi_hash) {
list_for_each_entry_safe(dp, next, &fp->fi_delegations, dl_perfile) {
deleg_func(dp);
if (++count == num)
return count;
}
}
}
return count;
}
void nfsd_forget_delegations(u64 num)
{
unsigned int count;
nfs4_lock_state();
count = nfsd_process_n_delegations(num, unhash_delegation);
nfs4_unlock_state();
printk(KERN_INFO "NFSD: Forgot %d delegations", count);
}
void nfsd_recall_delegations(u64 num)
{
unsigned int count;
nfs4_lock_state();
spin_lock(&recall_lock);
count = nfsd_process_n_delegations(num, nfsd_break_one_deleg);
spin_unlock(&recall_lock);
nfs4_unlock_state();
printk(KERN_INFO "NFSD: Recalled %d delegations", count);
}
#endif /* CONFIG_NFSD_FAULT_INJECTION */
/* initialization to perform at module load time: */ /* initialization to perform at module load time: */
int int
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "idmap.h" #include "idmap.h"
#include "nfsd.h" #include "nfsd.h"
#include "cache.h" #include "cache.h"
#include "fault_inject.h"
/* /*
* We have a single directory with several nodes in it. * We have a single directory with several nodes in it.
...@@ -1131,6 +1132,9 @@ static int __init init_nfsd(void) ...@@ -1131,6 +1132,9 @@ static int __init init_nfsd(void)
retval = nfs4_state_init(); /* nfs4 locking state */ retval = nfs4_state_init(); /* nfs4 locking state */
if (retval) if (retval)
return retval; return retval;
retval = nfsd_fault_inject_init(); /* nfsd fault injection controls */
if (retval)
goto out_free_slabs;
nfsd_stat_init(); /* Statistics */ nfsd_stat_init(); /* Statistics */
retval = nfsd_reply_cache_init(); retval = nfsd_reply_cache_init();
if (retval) if (retval)
...@@ -1161,6 +1165,8 @@ static int __init init_nfsd(void) ...@@ -1161,6 +1165,8 @@ static int __init init_nfsd(void)
nfsd_reply_cache_shutdown(); nfsd_reply_cache_shutdown();
out_free_stat: out_free_stat:
nfsd_stat_shutdown(); nfsd_stat_shutdown();
nfsd_fault_inject_cleanup();
out_free_slabs:
nfsd4_free_slabs(); nfsd4_free_slabs();
return retval; return retval;
} }
...@@ -1175,6 +1181,7 @@ static void __exit exit_nfsd(void) ...@@ -1175,6 +1181,7 @@ static void __exit exit_nfsd(void)
nfsd_lockd_shutdown(); nfsd_lockd_shutdown();
nfsd_idmap_shutdown(); nfsd_idmap_shutdown();
nfsd4_free_slabs(); nfsd4_free_slabs();
nfsd_fault_inject_cleanup();
unregister_filesystem(&nfsd_fs_type); unregister_filesystem(&nfsd_fs_type);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment