Commit 523b9458 authored by Christoph Lameter's avatar Christoph Lameter Committed by Linus Torvalds

Memoryless nodes: Fix GFP_THISNODE behavior

GFP_THISNODE checks that the zone selected is within the pgdat (node) of the
first zone of a nodelist.  That only works if the node has memory.  A
memoryless node will have its first node on another pgdat (node).

GFP_THISNODE currently will return simply memory on the first pgdat.  Thus it
is returning memory on other nodes.  GFP_THISNODE should fail if there is no
local memory on a node.

Add a new set of zonelists for each node that only contain the nodes that
belong to the zones itself so that no fallback is possible.

Then modify gfp_type to pickup the right zone based on the presence of
__GFP_THISNODE.

Drop the existing GFP_THISNODE checks from the page_allocators hot path.
Signed-off-by: default avatarChristoph Lameter <clameter@sgi.com>
Acked-by: default avatarNishanth Aravamudan <nacc@us.ibm.com>
Tested-by: default avatarLee Schermerhorn <lee.schermerhorn@hp.com>
Acked-by: default avatarBob Picco <bob.picco@hp.com>
Cc: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com>
Cc: Mel Gorman <mel@skynet.ie>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 633c0666
...@@ -98,22 +98,29 @@ struct vm_area_struct; ...@@ -98,22 +98,29 @@ struct vm_area_struct;
static inline enum zone_type gfp_zone(gfp_t flags) static inline enum zone_type gfp_zone(gfp_t flags)
{ {
int base = 0;
#ifdef CONFIG_NUMA
if (flags & __GFP_THISNODE)
base = MAX_NR_ZONES;
#endif
#ifdef CONFIG_ZONE_DMA #ifdef CONFIG_ZONE_DMA
if (flags & __GFP_DMA) if (flags & __GFP_DMA)
return ZONE_DMA; return base + ZONE_DMA;
#endif #endif
#ifdef CONFIG_ZONE_DMA32 #ifdef CONFIG_ZONE_DMA32
if (flags & __GFP_DMA32) if (flags & __GFP_DMA32)
return ZONE_DMA32; return base + ZONE_DMA32;
#endif #endif
if ((flags & (__GFP_HIGHMEM | __GFP_MOVABLE)) == if ((flags & (__GFP_HIGHMEM | __GFP_MOVABLE)) ==
(__GFP_HIGHMEM | __GFP_MOVABLE)) (__GFP_HIGHMEM | __GFP_MOVABLE))
return ZONE_MOVABLE; return base + ZONE_MOVABLE;
#ifdef CONFIG_HIGHMEM #ifdef CONFIG_HIGHMEM
if (flags & __GFP_HIGHMEM) if (flags & __GFP_HIGHMEM)
return ZONE_HIGHMEM; return base + ZONE_HIGHMEM;
#endif #endif
return ZONE_NORMAL; return base + ZONE_NORMAL;
} }
/* /*
......
...@@ -324,6 +324,17 @@ struct zone { ...@@ -324,6 +324,17 @@ struct zone {
#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES) #define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
/*
* The NUMA zonelists are doubled becausse we need zonelists that restrict the
* allocations to a single node for GFP_THISNODE.
*
* [0 .. MAX_NR_ZONES -1] : Zonelists with fallback
* [MAZ_NR_ZONES ... MAZ_ZONELISTS -1] : No fallback (GFP_THISNODE)
*/
#define MAX_ZONELISTS (2 * MAX_NR_ZONES)
/* /*
* We cache key information from each zonelist for smaller cache * We cache key information from each zonelist for smaller cache
* footprint when scanning for free pages in get_page_from_freelist(). * footprint when scanning for free pages in get_page_from_freelist().
...@@ -389,6 +400,7 @@ struct zonelist_cache { ...@@ -389,6 +400,7 @@ struct zonelist_cache {
unsigned long last_full_zap; /* when last zap'd (jiffies) */ unsigned long last_full_zap; /* when last zap'd (jiffies) */
}; };
#else #else
#define MAX_ZONELISTS MAX_NR_ZONES
struct zonelist_cache; struct zonelist_cache;
#endif #endif
...@@ -455,7 +467,7 @@ extern struct page *mem_map; ...@@ -455,7 +467,7 @@ extern struct page *mem_map;
struct bootmem_data; struct bootmem_data;
typedef struct pglist_data { typedef struct pglist_data {
struct zone node_zones[MAX_NR_ZONES]; struct zone node_zones[MAX_NR_ZONES];
struct zonelist node_zonelists[MAX_NR_ZONES]; struct zonelist node_zonelists[MAX_ZONELISTS];
int nr_zones; int nr_zones;
#ifdef CONFIG_FLAT_NODE_MEM_MAP #ifdef CONFIG_FLAT_NODE_MEM_MAP
struct page *node_mem_map; struct page *node_mem_map;
......
...@@ -1191,9 +1191,6 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order, ...@@ -1191,9 +1191,6 @@ get_page_from_freelist(gfp_t gfp_mask, unsigned int order,
!zlc_zone_worth_trying(zonelist, z, allowednodes)) !zlc_zone_worth_trying(zonelist, z, allowednodes))
continue; continue;
zone = *z; zone = *z;
if (unlikely(NUMA_BUILD && (gfp_mask & __GFP_THISNODE) &&
zone->zone_pgdat != zonelist->zones[0]->zone_pgdat))
break;
if ((alloc_flags & ALLOC_CPUSET) && if ((alloc_flags & ALLOC_CPUSET) &&
!cpuset_zone_allowed_softwall(zone, gfp_mask)) !cpuset_zone_allowed_softwall(zone, gfp_mask))
goto try_next_zone; goto try_next_zone;
...@@ -1262,7 +1259,10 @@ __alloc_pages(gfp_t gfp_mask, unsigned int order, ...@@ -1262,7 +1259,10 @@ __alloc_pages(gfp_t gfp_mask, unsigned int order,
z = zonelist->zones; /* the list of zones suitable for gfp_mask */ z = zonelist->zones; /* the list of zones suitable for gfp_mask */
if (unlikely(*z == NULL)) { if (unlikely(*z == NULL)) {
/* Should this ever happen?? */ /*
* Happens if we have an empty zonelist as a result of
* GFP_THISNODE being used on a memoryless node
*/
return NULL; return NULL;
} }
...@@ -1857,6 +1857,22 @@ static void build_zonelists_in_node_order(pg_data_t *pgdat, int node) ...@@ -1857,6 +1857,22 @@ static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
} }
} }
/*
* Build gfp_thisnode zonelists
*/
static void build_thisnode_zonelists(pg_data_t *pgdat)
{
enum zone_type i;
int j;
struct zonelist *zonelist;
for (i = 0; i < MAX_NR_ZONES; i++) {
zonelist = pgdat->node_zonelists + MAX_NR_ZONES + i;
j = build_zonelists_node(pgdat, zonelist, 0, i);
zonelist->zones[j] = NULL;
}
}
/* /*
* Build zonelists ordered by zone and nodes within zones. * Build zonelists ordered by zone and nodes within zones.
* This results in conserving DMA zone[s] until all Normal memory is * This results in conserving DMA zone[s] until all Normal memory is
...@@ -1961,7 +1977,7 @@ static void build_zonelists(pg_data_t *pgdat) ...@@ -1961,7 +1977,7 @@ static void build_zonelists(pg_data_t *pgdat)
int order = current_zonelist_order; int order = current_zonelist_order;
/* initialize zonelists */ /* initialize zonelists */
for (i = 0; i < MAX_NR_ZONES; i++) { for (i = 0; i < MAX_ZONELISTS; i++) {
zonelist = pgdat->node_zonelists + i; zonelist = pgdat->node_zonelists + i;
zonelist->zones[0] = NULL; zonelist->zones[0] = NULL;
} }
...@@ -2006,6 +2022,8 @@ static void build_zonelists(pg_data_t *pgdat) ...@@ -2006,6 +2022,8 @@ static void build_zonelists(pg_data_t *pgdat)
/* calculate node order -- i.e., DMA last! */ /* calculate node order -- i.e., DMA last! */
build_zonelists_in_zone_order(pgdat, j); build_zonelists_in_zone_order(pgdat, j);
} }
build_thisnode_zonelists(pgdat);
} }
/* Construct the zonelist performance cache - see further mmzone.h */ /* Construct the zonelist performance cache - see further mmzone.h */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment