Commit fc36b8d3 authored by Lee Schermerhorn's avatar Lee Schermerhorn Committed by Linus Torvalds

mempolicy: use MPOL_F_LOCAL to Indicate Preferred Local Policy

Now that we're using "preferred local" policy for system default, we need to
make this as fast as possible.  Because of the variable size of the mempolicy
structure [based on size of nodemasks], the preferred_node may be in a
different cacheline from the mode.  This can result in accessing an extra
cacheline in the normal case of system default policy.  Suspect this is the
cause of an observed 2-3% slowdown in page fault testing relative to kernel
without this patch series.

To alleviate this, use an internal mode flag, MPOL_F_LOCAL in the mempolicy
flags member which is guaranteed [?] to be in the same cacheline as the mode
itself.

Verified that reworked mempolicy now performs slightly better on 25-rc8-mm1
for both anon and shmem segments with system default and vma [preferred local]
policy.
Signed-off-by: default avatarLee Schermerhorn <lee.schermerhorn@hp.com>
Cc: Christoph Lameter <clameter@sgi.com>
Cc: David Rientjes <rientjes@google.com>
Cc: Mel Gorman <mel@csn.ul.ie>
Cc: Andi Kleen <ak@suse.de>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 53f2556b
...@@ -176,12 +176,11 @@ Components of Memory Policies ...@@ -176,12 +176,11 @@ Components of Memory Policies
containing the cpu where the allocation takes place. containing the cpu where the allocation takes place.
Internally, the Preferred policy uses a single node--the Internally, the Preferred policy uses a single node--the
preferred_node member of struct mempolicy. A "distinguished preferred_node member of struct mempolicy. When the internal
value of this preferred_node, currently '-1', is interpreted mode flag MPOL_F_LOCAL is set, the preferred_node is ignored and
as "the node containing the cpu where the allocation takes the policy is interpreted as local allocation. "Local" allocation
place"--local allocation. "Local" allocation policy can be policy can be viewed as a Preferred policy that starts at the node
viewed as a Preferred policy that starts at the node containing containing the cpu where the allocation takes place.
the cpu where the allocation takes place.
It is possible for the user to specify that local allocation is It is possible for the user to specify that local allocation is
always preferred by passing an empty nodemask with this mode. always preferred by passing an empty nodemask with this mode.
......
...@@ -50,6 +50,7 @@ enum { ...@@ -50,6 +50,7 @@ enum {
* are never OR'ed into the mode in mempolicy API arguments. * are never OR'ed into the mode in mempolicy API arguments.
*/ */
#define MPOL_F_SHARED (1 << 0) /* identify shared policies */ #define MPOL_F_SHARED (1 << 0) /* identify shared policies */
#define MPOL_F_LOCAL (1 << 1) /* preferred local allocation */
#ifdef __KERNEL__ #ifdef __KERNEL__
......
...@@ -110,7 +110,7 @@ enum zone_type policy_zone = 0; ...@@ -110,7 +110,7 @@ enum zone_type policy_zone = 0;
struct mempolicy default_policy = { struct mempolicy default_policy = {
.refcnt = ATOMIC_INIT(1), /* never free it */ .refcnt = ATOMIC_INIT(1), /* never free it */
.mode = MPOL_PREFERRED, .mode = MPOL_PREFERRED,
.v = { .preferred_node = -1 }, .flags = MPOL_F_LOCAL,
}; };
static const struct mempolicy_operations { static const struct mempolicy_operations {
...@@ -163,7 +163,7 @@ static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes) ...@@ -163,7 +163,7 @@ static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes) static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
{ {
if (!nodes) if (!nodes)
pol->v.preferred_node = -1; /* local allocation */ pol->flags |= MPOL_F_LOCAL; /* local allocation */
else if (nodes_empty(*nodes)) else if (nodes_empty(*nodes))
return -EINVAL; /* no allowed nodes */ return -EINVAL; /* no allowed nodes */
else else
...@@ -290,14 +290,15 @@ static void mpol_rebind_preferred(struct mempolicy *pol, ...@@ -290,14 +290,15 @@ static void mpol_rebind_preferred(struct mempolicy *pol,
if (pol->flags & MPOL_F_STATIC_NODES) { if (pol->flags & MPOL_F_STATIC_NODES) {
int node = first_node(pol->w.user_nodemask); int node = first_node(pol->w.user_nodemask);
if (node_isset(node, *nodes)) if (node_isset(node, *nodes)) {
pol->v.preferred_node = node; pol->v.preferred_node = node;
else pol->flags &= ~MPOL_F_LOCAL;
pol->v.preferred_node = -1; } else
pol->flags |= MPOL_F_LOCAL;
} else if (pol->flags & MPOL_F_RELATIVE_NODES) { } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes); mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
pol->v.preferred_node = first_node(tmp); pol->v.preferred_node = first_node(tmp);
} else if (pol->v.preferred_node != -1) { } else if (!(pol->flags & MPOL_F_LOCAL)) {
pol->v.preferred_node = node_remap(pol->v.preferred_node, pol->v.preferred_node = node_remap(pol->v.preferred_node,
pol->w.cpuset_mems_allowed, pol->w.cpuset_mems_allowed,
*nodes); *nodes);
...@@ -645,7 +646,7 @@ static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes) ...@@ -645,7 +646,7 @@ static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)
*nodes = p->v.nodes; *nodes = p->v.nodes;
break; break;
case MPOL_PREFERRED: case MPOL_PREFERRED:
if (p->v.preferred_node >= 0) if (!(p->flags & MPOL_F_LOCAL))
node_set(p->v.preferred_node, *nodes); node_set(p->v.preferred_node, *nodes);
/* else return empty node mask for local allocation */ /* else return empty node mask for local allocation */
break; break;
...@@ -1324,13 +1325,12 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy) ...@@ -1324,13 +1325,12 @@ static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
/* Return a zonelist indicated by gfp for node representing a mempolicy */ /* Return a zonelist indicated by gfp for node representing a mempolicy */
static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy) static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)
{ {
int nd; int nd = numa_node_id();
switch (policy->mode) { switch (policy->mode) {
case MPOL_PREFERRED: case MPOL_PREFERRED:
nd = policy->v.preferred_node; if (!(policy->flags & MPOL_F_LOCAL))
if (nd < 0) nd = policy->v.preferred_node;
nd = numa_node_id();
break; break;
case MPOL_BIND: case MPOL_BIND:
/* /*
...@@ -1339,16 +1339,13 @@ static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy) ...@@ -1339,16 +1339,13 @@ static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)
* current node is part of the mask, we use the zonelist for * current node is part of the mask, we use the zonelist for
* the first node in the mask instead. * the first node in the mask instead.
*/ */
nd = numa_node_id();
if (unlikely(gfp & __GFP_THISNODE) && if (unlikely(gfp & __GFP_THISNODE) &&
unlikely(!node_isset(nd, policy->v.nodes))) unlikely(!node_isset(nd, policy->v.nodes)))
nd = first_node(policy->v.nodes); nd = first_node(policy->v.nodes);
break; break;
case MPOL_INTERLEAVE: /* should not happen */ case MPOL_INTERLEAVE: /* should not happen */
nd = numa_node_id();
break; break;
default: default:
nd = 0;
BUG(); BUG();
} }
return node_zonelist(nd, gfp); return node_zonelist(nd, gfp);
...@@ -1379,14 +1376,15 @@ static unsigned interleave_nodes(struct mempolicy *policy) ...@@ -1379,14 +1376,15 @@ static unsigned interleave_nodes(struct mempolicy *policy)
*/ */
unsigned slab_node(struct mempolicy *policy) unsigned slab_node(struct mempolicy *policy)
{ {
if (!policy) if (!policy || policy->flags & MPOL_F_LOCAL)
return numa_node_id(); return numa_node_id();
switch (policy->mode) { switch (policy->mode) {
case MPOL_PREFERRED: case MPOL_PREFERRED:
if (unlikely(policy->v.preferred_node >= 0)) /*
return policy->v.preferred_node; * handled MPOL_F_LOCAL above
return numa_node_id(); */
return policy->v.preferred_node;
case MPOL_INTERLEAVE: case MPOL_INTERLEAVE:
return interleave_nodes(policy); return interleave_nodes(policy);
...@@ -1666,7 +1664,8 @@ int __mpol_equal(struct mempolicy *a, struct mempolicy *b) ...@@ -1666,7 +1664,8 @@ int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
case MPOL_INTERLEAVE: case MPOL_INTERLEAVE:
return nodes_equal(a->v.nodes, b->v.nodes); return nodes_equal(a->v.nodes, b->v.nodes);
case MPOL_PREFERRED: case MPOL_PREFERRED:
return a->v.preferred_node == b->v.preferred_node; return a->v.preferred_node == b->v.preferred_node &&
a->flags == b->flags;
default: default:
BUG(); BUG();
return 0; return 0;
...@@ -1946,7 +1945,7 @@ void numa_default_policy(void) ...@@ -1946,7 +1945,7 @@ void numa_default_policy(void)
} }
/* /*
* "local" is pseudo-policy: MPOL_PREFERRED with preferred_node == -1 * "local" is pseudo-policy: MPOL_PREFERRED with MPOL_F_LOCAL flag
* Used only for mpol_to_str() * Used only for mpol_to_str()
*/ */
#define MPOL_LOCAL (MPOL_INTERLEAVE + 1) #define MPOL_LOCAL (MPOL_INTERLEAVE + 1)
...@@ -1962,7 +1961,6 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol) ...@@ -1962,7 +1961,6 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
{ {
char *p = buffer; char *p = buffer;
int l; int l;
int nid;
nodemask_t nodes; nodemask_t nodes;
unsigned short mode; unsigned short mode;
unsigned short flags = pol ? pol->flags : 0; unsigned short flags = pol ? pol->flags : 0;
...@@ -1979,11 +1977,10 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol) ...@@ -1979,11 +1977,10 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
case MPOL_PREFERRED: case MPOL_PREFERRED:
nodes_clear(nodes); nodes_clear(nodes);
nid = pol->v.preferred_node; if (flags & MPOL_F_LOCAL)
if (nid < 0)
mode = MPOL_LOCAL; /* pseudo-policy */ mode = MPOL_LOCAL; /* pseudo-policy */
else else
node_set(nid, nodes); node_set(pol->v.preferred_node, nodes);
break; break;
case MPOL_BIND: case MPOL_BIND:
...@@ -2004,7 +2001,7 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol) ...@@ -2004,7 +2001,7 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
strcpy(p, policy_types[mode]); strcpy(p, policy_types[mode]);
p += l; p += l;
if (flags) { if (flags & MPOL_MODE_FLAGS) {
int need_bar = 0; int need_bar = 0;
if (buffer + maxlen < p + 2) if (buffer + maxlen < p + 2)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment