Commit c2452f32 authored by Nick Piggin's avatar Nick Piggin Committed by Al Viro

shrink struct dentry

struct dentry is one of the most critical structures in the kernel. So it's
sad to see it going neglected.

With CONFIG_PROFILING turned on (which is probably the common case at least
for distros and kernel developers), sizeof(struct dcache) == 208 here
(64-bit). This gives 19 objects per slab.

I packed d_mounted into a hole, and took another 4 bytes off the inline
name length to take the padding out from the end of the structure. This
shinks it to 200 bytes. I could have gone the other way and increased the
length to 40, but I'm aiming for a magic number, read on...

I then got rid of the d_cookie pointer. This shrinks it to 192 bytes. Rant:
why was this ever a good idea? The cookie system should increase its hash
size or use a tree or something if lookups are a problem. Also the "fast
dcookie lookups" in oprofile should be moved into the dcookie code -- how
can oprofile possibly care about the dcookie_mutex? It gets dropped after
get_dcookie() returns so it can't be providing any sort of protection.

At 192 bytes, 21 objects fit into a 4K page, saving about 3MB on my system
with ~140 000 entries allocated. 192 is also a multiple of 64, so we get
nice cacheline alignment on 64 and 32 byte line systems -- any given dentry
will now require 3 cachelines to touch all fields wheras previously it
would require 4.

I know the inline name size was chosen quite carefully, however with the
reduction in cacheline footprint, it should actually be just about as fast
to do a name lookup for a 36 character name as it was before the patch (and
faster for other sizes). The memory footprint savings for names which are
<= 32 or > 36 bytes long should more than make up for the memory cost for
33-36 byte names.

Performance is a feature...
Signed-off-by: default avatarAl Viro <viro@zeniv.linux.org.uk>
parent e2b689d8
...@@ -297,7 +297,7 @@ static inline unsigned long fast_get_dcookie(struct path *path) ...@@ -297,7 +297,7 @@ static inline unsigned long fast_get_dcookie(struct path *path)
{ {
unsigned long cookie; unsigned long cookie;
if (path->dentry->d_cookie) if (path->dentry->d_flags & DCACHE_COOKIE)
return (unsigned long)path->dentry; return (unsigned long)path->dentry;
get_dcookie(path, &cookie); get_dcookie(path, &cookie);
return cookie; return cookie;
......
...@@ -200,7 +200,7 @@ static inline unsigned long fast_get_dcookie(struct path *path) ...@@ -200,7 +200,7 @@ static inline unsigned long fast_get_dcookie(struct path *path)
{ {
unsigned long cookie; unsigned long cookie;
if (path->dentry->d_cookie) if (path->dentry->d_flags & DCACHE_COOKIE)
return (unsigned long)path->dentry; return (unsigned long)path->dentry;
get_dcookie(path, &cookie); get_dcookie(path, &cookie);
return cookie; return cookie;
......
...@@ -34,7 +34,6 @@ ...@@ -34,7 +34,6 @@
#include <linux/bootmem.h> #include <linux/bootmem.h>
#include "internal.h" #include "internal.h"
int sysctl_vfs_cache_pressure __read_mostly = 100; int sysctl_vfs_cache_pressure __read_mostly = 100;
EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
...@@ -948,9 +947,6 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) ...@@ -948,9 +947,6 @@ struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
dentry->d_op = NULL; dentry->d_op = NULL;
dentry->d_fsdata = NULL; dentry->d_fsdata = NULL;
dentry->d_mounted = 0; dentry->d_mounted = 0;
#ifdef CONFIG_PROFILING
dentry->d_cookie = NULL;
#endif
INIT_HLIST_NODE(&dentry->d_hash); INIT_HLIST_NODE(&dentry->d_hash);
INIT_LIST_HEAD(&dentry->d_lru); INIT_LIST_HEAD(&dentry->d_lru);
INIT_LIST_HEAD(&dentry->d_subdirs); INIT_LIST_HEAD(&dentry->d_subdirs);
......
...@@ -93,10 +93,15 @@ static struct dcookie_struct *alloc_dcookie(struct path *path) ...@@ -93,10 +93,15 @@ static struct dcookie_struct *alloc_dcookie(struct path *path)
{ {
struct dcookie_struct *dcs = kmem_cache_alloc(dcookie_cache, struct dcookie_struct *dcs = kmem_cache_alloc(dcookie_cache,
GFP_KERNEL); GFP_KERNEL);
struct dentry *d;
if (!dcs) if (!dcs)
return NULL; return NULL;
path->dentry->d_cookie = dcs; d = path->dentry;
spin_lock(&d->d_lock);
d->d_flags |= DCACHE_COOKIE;
spin_unlock(&d->d_lock);
dcs->path = *path; dcs->path = *path;
path_get(path); path_get(path);
hash_dcookie(dcs); hash_dcookie(dcs);
...@@ -119,14 +124,14 @@ int get_dcookie(struct path *path, unsigned long *cookie) ...@@ -119,14 +124,14 @@ int get_dcookie(struct path *path, unsigned long *cookie)
goto out; goto out;
} }
dcs = path->dentry->d_cookie; if (path->dentry->d_flags & DCACHE_COOKIE) {
dcs = find_dcookie((unsigned long)path->dentry);
if (!dcs) } else {
dcs = alloc_dcookie(path); dcs = alloc_dcookie(path);
if (!dcs) {
if (!dcs) { err = -ENOMEM;
err = -ENOMEM; goto out;
goto out; }
} }
*cookie = dcookie_value(dcs); *cookie = dcookie_value(dcs);
...@@ -251,7 +256,12 @@ static int dcookie_init(void) ...@@ -251,7 +256,12 @@ static int dcookie_init(void)
static void free_dcookie(struct dcookie_struct * dcs) static void free_dcookie(struct dcookie_struct * dcs)
{ {
dcs->path.dentry->d_cookie = NULL; struct dentry *d = dcs->path.dentry;
spin_lock(&d->d_lock);
d->d_flags &= ~DCACHE_COOKIE;
spin_unlock(&d->d_lock);
path_put(&dcs->path); path_put(&dcs->path);
kmem_cache_free(dcookie_cache, dcs); kmem_cache_free(dcookie_cache, dcs);
} }
......
...@@ -75,14 +75,22 @@ full_name_hash(const unsigned char *name, unsigned int len) ...@@ -75,14 +75,22 @@ full_name_hash(const unsigned char *name, unsigned int len)
return end_name_hash(hash); return end_name_hash(hash);
} }
struct dcookie_struct; /*
* Try to keep struct dentry aligned on 64 byte cachelines (this will
#define DNAME_INLINE_LEN_MIN 36 * give reasonable cacheline footprint with larger lines without the
* large memory footprint increase).
*/
#ifdef CONFIG_64BIT
#define DNAME_INLINE_LEN_MIN 32 /* 192 bytes */
#else
#define DNAME_INLINE_LEN_MIN 40 /* 128 bytes */
#endif
struct dentry { struct dentry {
atomic_t d_count; atomic_t d_count;
unsigned int d_flags; /* protected by d_lock */ unsigned int d_flags; /* protected by d_lock */
spinlock_t d_lock; /* per dentry lock */ spinlock_t d_lock; /* per dentry lock */
int d_mounted;
struct inode *d_inode; /* Where the name belongs to - NULL is struct inode *d_inode; /* Where the name belongs to - NULL is
* negative */ * negative */
/* /*
...@@ -107,10 +115,7 @@ struct dentry { ...@@ -107,10 +115,7 @@ struct dentry {
struct dentry_operations *d_op; struct dentry_operations *d_op;
struct super_block *d_sb; /* The root of the dentry tree */ struct super_block *d_sb; /* The root of the dentry tree */
void *d_fsdata; /* fs-specific data */ void *d_fsdata; /* fs-specific data */
#ifdef CONFIG_PROFILING
struct dcookie_struct *d_cookie; /* cookie, if any */
#endif
int d_mounted;
unsigned char d_iname[DNAME_INLINE_LEN_MIN]; /* small names */ unsigned char d_iname[DNAME_INLINE_LEN_MIN]; /* small names */
}; };
...@@ -177,6 +182,8 @@ d_iput: no no no yes ...@@ -177,6 +182,8 @@ d_iput: no no no yes
#define DCACHE_INOTIFY_PARENT_WATCHED 0x0020 /* Parent inode is watched */ #define DCACHE_INOTIFY_PARENT_WATCHED 0x0020 /* Parent inode is watched */
#define DCACHE_COOKIE 0x0040 /* For use by dcookie subsystem */
extern spinlock_t dcache_lock; extern spinlock_t dcache_lock;
extern seqlock_t rename_lock; extern seqlock_t rename_lock;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment