Commit ae2a8236 authored by Linus Torvalds's avatar Linus Torvalds

dcache: move the DCACHE_OP_COMPARE case out of the __d_lookup_rcu loop

__d_lookup_rcu() is one of the hottest functions in the kernel on
certain loads, and it is complicated by filesystems that might want to
have their own name compare function.

We can improve code generation by moving the test of DCACHE_OP_COMPARE
outside the loop, which makes the loop itself much simpler, at the cost
of some code duplication.  But both cases end up being simpler, and the
"native" direct case-sensitive compare particularly so.

Cc: Al Viro <viro@zeniv.linux.org.uk>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 274a2eeb
...@@ -2270,6 +2270,48 @@ bool d_same_name(const struct dentry *dentry, const struct dentry *parent, ...@@ -2270,6 +2270,48 @@ bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
} }
EXPORT_SYMBOL_GPL(d_same_name); EXPORT_SYMBOL_GPL(d_same_name);
/*
* This is __d_lookup_rcu() when the parent dentry has
* DCACHE_OP_COMPARE, which makes things much nastier.
*/
static noinline struct dentry *__d_lookup_rcu_op_compare(
const struct dentry *parent,
const struct qstr *name,
unsigned *seqp)
{
u64 hashlen = name->hash_len;
struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
struct hlist_bl_node *node;
struct dentry *dentry;
hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
int tlen;
const char *tname;
unsigned seq;
seqretry:
seq = raw_seqcount_begin(&dentry->d_seq);
if (dentry->d_parent != parent)
continue;
if (d_unhashed(dentry))
continue;
if (dentry->d_name.hash != hashlen_hash(hashlen))
continue;
tlen = dentry->d_name.len;
tname = dentry->d_name.name;
/* we want a consistent (name,len) pair */
if (read_seqcount_retry(&dentry->d_seq, seq)) {
cpu_relax();
goto seqretry;
}
if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0)
continue;
*seqp = seq;
return dentry;
}
return NULL;
}
/** /**
* __d_lookup_rcu - search for a dentry (racy, store-free) * __d_lookup_rcu - search for a dentry (racy, store-free)
* @parent: parent dentry * @parent: parent dentry
...@@ -2316,6 +2358,9 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, ...@@ -2316,6 +2358,9 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent,
* Keep the two functions in sync. * Keep the two functions in sync.
*/ */
if (unlikely(parent->d_flags & DCACHE_OP_COMPARE))
return __d_lookup_rcu_op_compare(parent, name, seqp);
/* /*
* The hash list is protected using RCU. * The hash list is protected using RCU.
* *
...@@ -2332,7 +2377,6 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, ...@@ -2332,7 +2377,6 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent,
hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
unsigned seq; unsigned seq;
seqretry:
/* /*
* The dentry sequence count protects us from concurrent * The dentry sequence count protects us from concurrent
* renames, and thus protects parent and name fields. * renames, and thus protects parent and name fields.
...@@ -2355,28 +2399,10 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent, ...@@ -2355,28 +2399,10 @@ struct dentry *__d_lookup_rcu(const struct dentry *parent,
continue; continue;
if (d_unhashed(dentry)) if (d_unhashed(dentry))
continue; continue;
if (dentry->d_name.hash_len != hashlen)
if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { continue;
int tlen; if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
const char *tname; continue;
if (dentry->d_name.hash != hashlen_hash(hashlen))
continue;
tlen = dentry->d_name.len;
tname = dentry->d_name.name;
/* we want a consistent (name,len) pair */
if (read_seqcount_retry(&dentry->d_seq, seq)) {
cpu_relax();
goto seqretry;
}
if (parent->d_op->d_compare(dentry,
tlen, tname, name) != 0)
continue;
} else {
if (dentry->d_name.hash_len != hashlen)
continue;
if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
continue;
}
*seqp = seq; *seqp = seq;
return dentry; return dentry;
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment