Commit 7b24d861 authored by Davidlohr Bueso's avatar Davidlohr Bueso Committed by Linus Torvalds

mm, hugetlb: fix race in region tracking

There is a race condition if we map a same file on different processes.
Region tracking is protected by mmap_sem and hugetlb_instantiation_mutex.
When we do mmap, we don't grab a hugetlb_instantiation_mutex, but only
mmap_sem (exclusively).  This doesn't prevent other tasks from modifying
the region structure, so it can be modified by two processes
concurrently.

To solve this, introduce a spinlock to resv_map and make region
manipulation function grab it before they do actual work.

[davidlohr@hp.com: updated changelog]
Signed-off-by: default avatarDavidlohr Bueso <davidlohr@hp.com>
Signed-off-by: default avatarJoonsoo Kim <iamjoonsoo.kim@lge.com>
Suggested-by: default avatarJoonsoo Kim <iamjoonsoo.kim@lge.com>
Acked-by: default avatarDavid Gibson <david@gibson.dropbear.id.au>
Cc: David Gibson <david@gibson.dropbear.id.au>
Cc: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent 1406ec9b
...@@ -27,6 +27,7 @@ struct hugepage_subpool { ...@@ -27,6 +27,7 @@ struct hugepage_subpool {
struct resv_map { struct resv_map {
struct kref refs; struct kref refs;
spinlock_t lock;
struct list_head regions; struct list_head regions;
}; };
extern struct resv_map *resv_map_alloc(void); extern struct resv_map *resv_map_alloc(void);
......
...@@ -135,15 +135,8 @@ static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma) ...@@ -135,15 +135,8 @@ static inline struct hugepage_subpool *subpool_vma(struct vm_area_struct *vma)
* Region tracking -- allows tracking of reservations and instantiated pages * Region tracking -- allows tracking of reservations and instantiated pages
* across the pages in a mapping. * across the pages in a mapping.
* *
* The region data structures are protected by a combination of the mmap_sem * The region data structures are embedded into a resv_map and
* and the hugetlb_instantiation_mutex. To access or modify a region the caller * protected by a resv_map's lock
* must either hold the mmap_sem for write, or the mmap_sem for read and
* the hugetlb_instantiation_mutex:
*
* down_write(&mm->mmap_sem);
* or
* down_read(&mm->mmap_sem);
* mutex_lock(&hugetlb_instantiation_mutex);
*/ */
struct file_region { struct file_region {
struct list_head link; struct list_head link;
...@@ -156,6 +149,7 @@ static long region_add(struct resv_map *resv, long f, long t) ...@@ -156,6 +149,7 @@ static long region_add(struct resv_map *resv, long f, long t)
struct list_head *head = &resv->regions; struct list_head *head = &resv->regions;
struct file_region *rg, *nrg, *trg; struct file_region *rg, *nrg, *trg;
spin_lock(&resv->lock);
/* Locate the region we are either in or before. */ /* Locate the region we are either in or before. */
list_for_each_entry(rg, head, link) list_for_each_entry(rg, head, link)
if (f <= rg->to) if (f <= rg->to)
...@@ -185,15 +179,18 @@ static long region_add(struct resv_map *resv, long f, long t) ...@@ -185,15 +179,18 @@ static long region_add(struct resv_map *resv, long f, long t)
} }
nrg->from = f; nrg->from = f;
nrg->to = t; nrg->to = t;
spin_unlock(&resv->lock);
return 0; return 0;
} }
static long region_chg(struct resv_map *resv, long f, long t) static long region_chg(struct resv_map *resv, long f, long t)
{ {
struct list_head *head = &resv->regions; struct list_head *head = &resv->regions;
struct file_region *rg, *nrg; struct file_region *rg, *nrg = NULL;
long chg = 0; long chg = 0;
retry:
spin_lock(&resv->lock);
/* Locate the region we are before or in. */ /* Locate the region we are before or in. */
list_for_each_entry(rg, head, link) list_for_each_entry(rg, head, link)
if (f <= rg->to) if (f <= rg->to)
...@@ -203,15 +200,21 @@ static long region_chg(struct resv_map *resv, long f, long t) ...@@ -203,15 +200,21 @@ static long region_chg(struct resv_map *resv, long f, long t)
* Subtle, allocate a new region at the position but make it zero * Subtle, allocate a new region at the position but make it zero
* size such that we can guarantee to record the reservation. */ * size such that we can guarantee to record the reservation. */
if (&rg->link == head || t < rg->from) { if (&rg->link == head || t < rg->from) {
nrg = kmalloc(sizeof(*nrg), GFP_KERNEL); if (!nrg) {
if (!nrg) spin_unlock(&resv->lock);
return -ENOMEM; nrg = kmalloc(sizeof(*nrg), GFP_KERNEL);
nrg->from = f; if (!nrg)
nrg->to = f; return -ENOMEM;
INIT_LIST_HEAD(&nrg->link);
list_add(&nrg->link, rg->link.prev); nrg->from = f;
nrg->to = f;
INIT_LIST_HEAD(&nrg->link);
goto retry;
}
return t - f; list_add(&nrg->link, rg->link.prev);
chg = t - f;
goto out_nrg;
} }
/* Round our left edge to the current segment if it encloses us. */ /* Round our left edge to the current segment if it encloses us. */
...@@ -224,7 +227,7 @@ static long region_chg(struct resv_map *resv, long f, long t) ...@@ -224,7 +227,7 @@ static long region_chg(struct resv_map *resv, long f, long t)
if (&rg->link == head) if (&rg->link == head)
break; break;
if (rg->from > t) if (rg->from > t)
return chg; goto out;
/* We overlap with this area, if it extends further than /* We overlap with this area, if it extends further than
* us then we must extend ourselves. Account for its * us then we must extend ourselves. Account for its
...@@ -235,6 +238,14 @@ static long region_chg(struct resv_map *resv, long f, long t) ...@@ -235,6 +238,14 @@ static long region_chg(struct resv_map *resv, long f, long t)
} }
chg -= rg->to - rg->from; chg -= rg->to - rg->from;
} }
out:
spin_unlock(&resv->lock);
/* We already know we raced and no longer need the new region */
kfree(nrg);
return chg;
out_nrg:
spin_unlock(&resv->lock);
return chg; return chg;
} }
...@@ -244,12 +255,13 @@ static long region_truncate(struct resv_map *resv, long end) ...@@ -244,12 +255,13 @@ static long region_truncate(struct resv_map *resv, long end)
struct file_region *rg, *trg; struct file_region *rg, *trg;
long chg = 0; long chg = 0;
spin_lock(&resv->lock);
/* Locate the region we are either in or before. */ /* Locate the region we are either in or before. */
list_for_each_entry(rg, head, link) list_for_each_entry(rg, head, link)
if (end <= rg->to) if (end <= rg->to)
break; break;
if (&rg->link == head) if (&rg->link == head)
return 0; goto out;
/* If we are in the middle of a region then adjust it. */ /* If we are in the middle of a region then adjust it. */
if (end > rg->from) { if (end > rg->from) {
...@@ -266,6 +278,9 @@ static long region_truncate(struct resv_map *resv, long end) ...@@ -266,6 +278,9 @@ static long region_truncate(struct resv_map *resv, long end)
list_del(&rg->link); list_del(&rg->link);
kfree(rg); kfree(rg);
} }
out:
spin_unlock(&resv->lock);
return chg; return chg;
} }
...@@ -275,6 +290,7 @@ static long region_count(struct resv_map *resv, long f, long t) ...@@ -275,6 +290,7 @@ static long region_count(struct resv_map *resv, long f, long t)
struct file_region *rg; struct file_region *rg;
long chg = 0; long chg = 0;
spin_lock(&resv->lock);
/* Locate each segment we overlap with, and count that overlap. */ /* Locate each segment we overlap with, and count that overlap. */
list_for_each_entry(rg, head, link) { list_for_each_entry(rg, head, link) {
long seg_from; long seg_from;
...@@ -290,6 +306,7 @@ static long region_count(struct resv_map *resv, long f, long t) ...@@ -290,6 +306,7 @@ static long region_count(struct resv_map *resv, long f, long t)
chg += seg_to - seg_from; chg += seg_to - seg_from;
} }
spin_unlock(&resv->lock);
return chg; return chg;
} }
...@@ -387,6 +404,7 @@ struct resv_map *resv_map_alloc(void) ...@@ -387,6 +404,7 @@ struct resv_map *resv_map_alloc(void)
return NULL; return NULL;
kref_init(&resv_map->refs); kref_init(&resv_map->refs);
spin_lock_init(&resv_map->lock);
INIT_LIST_HEAD(&resv_map->regions); INIT_LIST_HEAD(&resv_map->regions);
return resv_map; return resv_map;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment