Commit e77b0852 authored by Aneesh Kumar K.V's avatar Aneesh Kumar K.V Committed by Linus Torvalds

mm/mmu_gather: track page size with mmu gather and force flush if page size change

This allows an arch which needs to do special handing with respect to
different page size when flushing tlb to implement the same in mmu
gather.

Link: http://lkml.kernel.org/r/1465049193-22197-3-git-send-email-aneesh.kumar@linux.vnet.ibm.comSigned-off-by: default avatarAneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Hugh Dickins <hughd@google.com>
Cc: "Kirill A. Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Cc: Joonsoo Kim <iamjoonsoo.kim@lge.com>
Cc: Mel Gorman <mgorman@suse.de>
Cc: David Rientjes <rientjes@google.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Minchan Kim <minchan.kim@gmail.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@linux-foundation.org>
parent e9d55e15
...@@ -225,12 +225,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) ...@@ -225,12 +225,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
} }
} }
static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return __tlb_remove_page(tlb, page);
}
static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
struct page *page) struct page *page)
{ {
return __tlb_remove_page(tlb, page); return __tlb_remove_page(tlb, page);
} }
static inline void tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return tlb_remove_page(tlb, page);
}
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
unsigned long addr) unsigned long addr)
{ {
......
...@@ -242,12 +242,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) ...@@ -242,12 +242,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
} }
} }
static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return __tlb_remove_page(tlb, page);
}
static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
struct page *page) struct page *page)
{ {
return __tlb_remove_page(tlb, page); return __tlb_remove_page(tlb, page);
} }
static inline void tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return tlb_remove_page(tlb, page);
}
/* /*
* Remove TLB entry for PTE mapped at virtual address ADDRESS. This is called for any * Remove TLB entry for PTE mapped at virtual address ADDRESS. This is called for any
* PTE, not just those pointing to (normal) physical memory. * PTE, not just those pointing to (normal) physical memory.
......
...@@ -98,11 +98,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) ...@@ -98,11 +98,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
free_page_and_swap_cache(page); free_page_and_swap_cache(page);
} }
static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return __tlb_remove_page(tlb, page);
}
static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
struct page *page) struct page *page)
{ {
return __tlb_remove_page(tlb, page); return __tlb_remove_page(tlb, page);
} }
static inline void tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return tlb_remove_page(tlb, page);
}
/* /*
* pte_free_tlb frees a pte table and clears the CRSTE for the * pte_free_tlb frees a pte table and clears the CRSTE for the
* page table from the tlb. * page table from the tlb.
......
...@@ -109,12 +109,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) ...@@ -109,12 +109,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
__tlb_remove_page(tlb, page); __tlb_remove_page(tlb, page);
} }
static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return __tlb_remove_page(tlb, page);
}
static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
struct page *page) struct page *page)
{ {
return __tlb_remove_page(tlb, page); return __tlb_remove_page(tlb, page);
} }
static inline void tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return tlb_remove_page(tlb, page);
}
#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep) #define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep)
#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) #define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp) #define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
......
...@@ -110,12 +110,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) ...@@ -110,12 +110,24 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
__tlb_remove_page(tlb, page); __tlb_remove_page(tlb, page);
} }
static inline bool __tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return __tlb_remove_page(tlb, page);
}
static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb,
struct page *page) struct page *page)
{ {
return __tlb_remove_page(tlb, page); return __tlb_remove_page(tlb, page);
} }
static inline void tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
return tlb_remove_page(tlb, page);
}
/** /**
* tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation. * tlb_remove_tlb_entry - remember a pte unmapping for later tlb invalidation.
* *
......
...@@ -112,6 +112,7 @@ struct mmu_gather { ...@@ -112,6 +112,7 @@ struct mmu_gather {
* that that we can adjust the range after the flush * that that we can adjust the range after the flush
*/ */
unsigned long addr; unsigned long addr;
int page_size;
}; };
#define HAVE_GENERIC_MMU_GATHER #define HAVE_GENERIC_MMU_GATHER
...@@ -120,7 +121,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long ...@@ -120,7 +121,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long
void tlb_flush_mmu(struct mmu_gather *tlb); void tlb_flush_mmu(struct mmu_gather *tlb);
void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
unsigned long end); unsigned long end);
bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page); extern bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page,
int page_size);
static inline void __tlb_adjust_range(struct mmu_gather *tlb, static inline void __tlb_adjust_range(struct mmu_gather *tlb,
unsigned long address) unsigned long address)
...@@ -145,23 +147,36 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb) ...@@ -145,23 +147,36 @@ static inline void __tlb_reset_range(struct mmu_gather *tlb)
} }
} }
static inline void tlb_remove_page_size(struct mmu_gather *tlb,
struct page *page, int page_size)
{
if (__tlb_remove_page_size(tlb, page, page_size)) {
tlb_flush_mmu(tlb);
tlb->page_size = page_size;
__tlb_adjust_range(tlb, tlb->addr);
__tlb_remove_page_size(tlb, page, page_size);
}
}
static bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
return __tlb_remove_page_size(tlb, page, PAGE_SIZE);
}
/* tlb_remove_page /* tlb_remove_page
* Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when * Similar to __tlb_remove_page but will call tlb_flush_mmu() itself when
* required. * required.
*/ */
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{ {
if (__tlb_remove_page(tlb, page)) { return tlb_remove_page_size(tlb, page, PAGE_SIZE);
tlb_flush_mmu(tlb);
__tlb_adjust_range(tlb, tlb->addr);
__tlb_remove_page(tlb, page);
}
} }
static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, struct page *page) static inline bool __tlb_remove_pte_page(struct mmu_gather *tlb, struct page *page)
{ {
/* active->nr should be zero when we call this */ /* active->nr should be zero when we call this */
VM_BUG_ON_PAGE(tlb->active->nr, page); VM_BUG_ON_PAGE(tlb->active->nr, page);
tlb->page_size = PAGE_SIZE;
__tlb_adjust_range(tlb, tlb->addr); __tlb_adjust_range(tlb, tlb->addr);
return __tlb_remove_page(tlb, page); return __tlb_remove_page(tlb, page);
} }
......
...@@ -1689,7 +1689,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, ...@@ -1689,7 +1689,7 @@ int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma,
pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd)); pte_free(tlb->mm, pgtable_trans_huge_withdraw(tlb->mm, pmd));
atomic_long_dec(&tlb->mm->nr_ptes); atomic_long_dec(&tlb->mm->nr_ptes);
spin_unlock(ptl); spin_unlock(ptl);
tlb_remove_page(tlb, page); tlb_remove_page_size(tlb, page, HPAGE_PMD_SIZE);
} }
return 1; return 1;
} }
......
...@@ -3250,7 +3250,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma, ...@@ -3250,7 +3250,7 @@ void __unmap_hugepage_range(struct mmu_gather *tlb, struct vm_area_struct *vma,
page_remove_rmap(page, true); page_remove_rmap(page, true);
spin_unlock(ptl); spin_unlock(ptl);
tlb_remove_page(tlb, page); tlb_remove_page_size(tlb, page, huge_page_size(h));
/* /*
* Bail out after unmapping reference page if supplied * Bail out after unmapping reference page if supplied
*/ */
......
...@@ -233,6 +233,7 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long ...@@ -233,6 +233,7 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned long
#ifdef CONFIG_HAVE_RCU_TABLE_FREE #ifdef CONFIG_HAVE_RCU_TABLE_FREE
tlb->batch = NULL; tlb->batch = NULL;
#endif #endif
tlb->page_size = 0;
__tlb_reset_range(tlb); __tlb_reset_range(tlb);
} }
...@@ -294,12 +295,19 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e ...@@ -294,12 +295,19 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
* When out of page slots we must call tlb_flush_mmu(). * When out of page slots we must call tlb_flush_mmu().
*returns true if the caller should flush. *returns true if the caller should flush.
*/ */
bool __tlb_remove_page(struct mmu_gather *tlb, struct page *page) bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
{ {
struct mmu_gather_batch *batch; struct mmu_gather_batch *batch;
VM_BUG_ON(!tlb->end); VM_BUG_ON(!tlb->end);
if (!tlb->page_size)
tlb->page_size = page_size;
else {
if (page_size != tlb->page_size)
return true;
}
batch = tlb->active; batch = tlb->active;
if (batch->nr == batch->max) { if (batch->nr == batch->max) {
if (!tlb_next_batch(tlb)) if (!tlb_next_batch(tlb))
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment