Commit cd94154c authored by Martin Schwidefsky's avatar Martin Schwidefsky

[S390] fix tlb flushing for page table pages

Git commit 36409f63 "use generic RCU
page-table freeing code" introduced a tlb flushing bug. Partially revert
the above git commit and go back to s390 specific page table flush code.

For s390 the TLB can contain three types of entries, "normal" TLB
page-table entries, TLB combined region-and-segment-table (CRST) entries
and real-space entries. Linux does not use real-space entries which
leaves normal TLB entries and CRST entries. The CRST entries are
intermediate steps in the page-table translation called translation paths.
For example a 4K page access in a three-level page table setup will
create two CRST TLB entries and one page-table TLB entry. The advantage
of that approach is that a page access next to the previous one can reuse
the CRST entries and needs just a single read from memory to create the
page-table TLB entry. The disadvantage is that the TLB flushing rules are
more complicated, before any page-table may be freed the TLB needs to be
flushed.

In short: the generic RCU page-table freeing code is incorrect for the
CRST entries, in particular the check for mm_users < 2 is troublesome.

This is applicable to 3.0+ kernels.

Cc: <stable@vger.kernel.org>
Signed-off-by: default avatarMartin Schwidefsky <schwidefsky@de.ibm.com>
parent b785e0d0
...@@ -90,7 +90,6 @@ config S390 ...@@ -90,7 +90,6 @@ config S390
select HAVE_KERNEL_XZ select HAVE_KERNEL_XZ
select HAVE_ARCH_MUTEX_CPU_RELAX select HAVE_ARCH_MUTEX_CPU_RELAX
select HAVE_ARCH_JUMP_LABEL if !MARCH_G5 select HAVE_ARCH_JUMP_LABEL if !MARCH_G5
select HAVE_RCU_TABLE_FREE if SMP
select ARCH_SAVE_PAGE_KEYS if HIBERNATION select ARCH_SAVE_PAGE_KEYS if HIBERNATION
select HAVE_MEMBLOCK select HAVE_MEMBLOCK
select HAVE_MEMBLOCK_NODE_MAP select HAVE_MEMBLOCK_NODE_MAP
......
...@@ -22,10 +22,7 @@ void crst_table_free(struct mm_struct *, unsigned long *); ...@@ -22,10 +22,7 @@ void crst_table_free(struct mm_struct *, unsigned long *);
unsigned long *page_table_alloc(struct mm_struct *, unsigned long); unsigned long *page_table_alloc(struct mm_struct *, unsigned long);
void page_table_free(struct mm_struct *, unsigned long *); void page_table_free(struct mm_struct *, unsigned long *);
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
void page_table_free_rcu(struct mmu_gather *, unsigned long *); void page_table_free_rcu(struct mmu_gather *, unsigned long *);
void __tlb_remove_table(void *_table);
#endif
static inline void clear_table(unsigned long *s, unsigned long val, size_t n) static inline void clear_table(unsigned long *s, unsigned long val, size_t n)
{ {
......
...@@ -30,14 +30,10 @@ ...@@ -30,14 +30,10 @@
struct mmu_gather { struct mmu_gather {
struct mm_struct *mm; struct mm_struct *mm;
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
struct mmu_table_batch *batch; struct mmu_table_batch *batch;
#endif
unsigned int fullmm; unsigned int fullmm;
unsigned int need_flush;
}; };
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
struct mmu_table_batch { struct mmu_table_batch {
struct rcu_head rcu; struct rcu_head rcu;
unsigned int nr; unsigned int nr;
...@@ -49,7 +45,6 @@ struct mmu_table_batch { ...@@ -49,7 +45,6 @@ struct mmu_table_batch {
extern void tlb_table_flush(struct mmu_gather *tlb); extern void tlb_table_flush(struct mmu_gather *tlb);
extern void tlb_remove_table(struct mmu_gather *tlb, void *table); extern void tlb_remove_table(struct mmu_gather *tlb, void *table);
#endif
static inline void tlb_gather_mmu(struct mmu_gather *tlb, static inline void tlb_gather_mmu(struct mmu_gather *tlb,
struct mm_struct *mm, struct mm_struct *mm,
...@@ -57,29 +52,20 @@ static inline void tlb_gather_mmu(struct mmu_gather *tlb, ...@@ -57,29 +52,20 @@ static inline void tlb_gather_mmu(struct mmu_gather *tlb,
{ {
tlb->mm = mm; tlb->mm = mm;
tlb->fullmm = full_mm_flush; tlb->fullmm = full_mm_flush;
tlb->need_flush = 0;
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
tlb->batch = NULL; tlb->batch = NULL;
#endif
if (tlb->fullmm) if (tlb->fullmm)
__tlb_flush_mm(mm); __tlb_flush_mm(mm);
} }
static inline void tlb_flush_mmu(struct mmu_gather *tlb) static inline void tlb_flush_mmu(struct mmu_gather *tlb)
{ {
if (!tlb->need_flush)
return;
tlb->need_flush = 0;
__tlb_flush_mm(tlb->mm);
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
tlb_table_flush(tlb); tlb_table_flush(tlb);
#endif
} }
static inline void tlb_finish_mmu(struct mmu_gather *tlb, static inline void tlb_finish_mmu(struct mmu_gather *tlb,
unsigned long start, unsigned long end) unsigned long start, unsigned long end)
{ {
tlb_flush_mmu(tlb); tlb_table_flush(tlb);
} }
/* /*
...@@ -105,10 +91,8 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page) ...@@ -105,10 +91,8 @@ static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, static inline void pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
unsigned long address) unsigned long address)
{ {
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
if (!tlb->fullmm) if (!tlb->fullmm)
return page_table_free_rcu(tlb, (unsigned long *) pte); return page_table_free_rcu(tlb, (unsigned long *) pte);
#endif
page_table_free(tlb->mm, (unsigned long *) pte); page_table_free(tlb->mm, (unsigned long *) pte);
} }
...@@ -125,10 +109,8 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, ...@@ -125,10 +109,8 @@ static inline void pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd,
#ifdef __s390x__ #ifdef __s390x__
if (tlb->mm->context.asce_limit <= (1UL << 31)) if (tlb->mm->context.asce_limit <= (1UL << 31))
return; return;
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
if (!tlb->fullmm) if (!tlb->fullmm)
return tlb_remove_table(tlb, pmd); return tlb_remove_table(tlb, pmd);
#endif
crst_table_free(tlb->mm, (unsigned long *) pmd); crst_table_free(tlb->mm, (unsigned long *) pmd);
#endif #endif
} }
...@@ -146,10 +128,8 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud, ...@@ -146,10 +128,8 @@ static inline void pud_free_tlb(struct mmu_gather *tlb, pud_t *pud,
#ifdef __s390x__ #ifdef __s390x__
if (tlb->mm->context.asce_limit <= (1UL << 42)) if (tlb->mm->context.asce_limit <= (1UL << 42))
return; return;
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
if (!tlb->fullmm) if (!tlb->fullmm)
return tlb_remove_table(tlb, pud); return tlb_remove_table(tlb, pud);
#endif
crst_table_free(tlb->mm, (unsigned long *) pud); crst_table_free(tlb->mm, (unsigned long *) pud);
#endif #endif
} }
......
...@@ -678,8 +678,6 @@ void page_table_free(struct mm_struct *mm, unsigned long *table) ...@@ -678,8 +678,6 @@ void page_table_free(struct mm_struct *mm, unsigned long *table)
} }
} }
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
static void __page_table_free_rcu(void *table, unsigned bit) static void __page_table_free_rcu(void *table, unsigned bit)
{ {
struct page *page; struct page *page;
...@@ -733,7 +731,66 @@ void __tlb_remove_table(void *_table) ...@@ -733,7 +731,66 @@ void __tlb_remove_table(void *_table)
free_pages((unsigned long) table, ALLOC_ORDER); free_pages((unsigned long) table, ALLOC_ORDER);
} }
#endif static void tlb_remove_table_smp_sync(void *arg)
{
/* Simply deliver the interrupt */
}
static void tlb_remove_table_one(void *table)
{
/*
* This isn't an RCU grace period and hence the page-tables cannot be
* assumed to be actually RCU-freed.
*
* It is however sufficient for software page-table walkers that rely
* on IRQ disabling. See the comment near struct mmu_table_batch.
*/
smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
__tlb_remove_table(table);
}
static void tlb_remove_table_rcu(struct rcu_head *head)
{
struct mmu_table_batch *batch;
int i;
batch = container_of(head, struct mmu_table_batch, rcu);
for (i = 0; i < batch->nr; i++)
__tlb_remove_table(batch->tables[i]);
free_page((unsigned long)batch);
}
void tlb_table_flush(struct mmu_gather *tlb)
{
struct mmu_table_batch **batch = &tlb->batch;
if (*batch) {
__tlb_flush_mm(tlb->mm);
call_rcu_sched(&(*batch)->rcu, tlb_remove_table_rcu);
*batch = NULL;
}
}
void tlb_remove_table(struct mmu_gather *tlb, void *table)
{
struct mmu_table_batch **batch = &tlb->batch;
if (*batch == NULL) {
*batch = (struct mmu_table_batch *)
__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
if (*batch == NULL) {
__tlb_flush_mm(tlb->mm);
tlb_remove_table_one(table);
return;
}
(*batch)->nr = 0;
}
(*batch)->tables[(*batch)->nr++] = table;
if ((*batch)->nr == MAX_TABLE_BATCH)
tlb_table_flush(tlb);
}
/* /*
* switch on pgstes for its userspace process (for kvm) * switch on pgstes for its userspace process (for kvm)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment