[S390] refactor page table functions for better pgste support

Rework the architecture page table functions to access the bits in the
page table extension array (pgste). There are a number of changes:
1) Fix missing pgste update if the attach_count for the mm is <= 1.
2) For every operation that affects the invalid bit in the pte or the
   rcp byte in the pgste the pcl lock needs to be acquired. The function
   pgste_get_lock gets the pcl lock and returns the current pgste value
   for a pte pointer. The function pgste_set_unlock stores the pgste
   and releases the lock. Between these two calls the bits in the pgste
   can be shuffled.
3) Define two software bits in the pte _PAGE_SWR and _PAGE_SWC to avoid
   calling SetPageDirty and SetPageReferenced from pgtable.h. If the
   host reference backup bit or the host change backup bit has been
   set the dirty/referenced state is transfered to the pte. The common
   code will pick up the state from the pte.
4) Add ptep_modify_prot_start and ptep_modify_prot_commit for mprotect.
5) Remove pgd_populate_kernel, pud_populate_kernel, pmd_populate_kernel
   pgd_clear_kernel, pud_clear_kernel, pmd_clear_kernel and ptep_invalidate.
6) Rename kvm_s390_test_and_clear_page_dirty to
   ptep_test_and_clear_user_dirty and add ptep_test_and_clear_user_young.
7) Define mm_exclusive() and mm_has_pgste() helper to improve readability.
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

arch/s390/include/asm/mmu.h

@@ -9,8 +9,10 @@ typedef struct {
 	unsigned long asce_bits;
 	unsigned long asce_limit;
 	unsigned long vdso_base;
-	int has_pgste;	 /* The mmu context has extended page tables */
-	int alloc_pgste; /* cloned contexts will have extended page tables */
+	/* Cloned contexts will be created with extended page tables. */
+	unsigned int alloc_pgste:1;
+	/* The mmu context has extended page tables. */
+	unsigned int has_pgste:1;
 } mm_context_t;
 
 #define INIT_MM_CONTEXT(name)						      \

arch/s390/include/asm/page.h

@@ -90,6 +90,7 @@ static inline void copy_page(void *to, void *from)
  */
 
 typedef struct { unsigned long pgprot; } pgprot_t;
+typedef struct { unsigned long pgste; } pgste_t;
 typedef struct { unsigned long pte; } pte_t;
 typedef struct { unsigned long pmd; } pmd_t;
 typedef struct { unsigned long pud; } pud_t;
@@ -97,13 +98,16 @@ typedef struct { unsigned long pgd; } pgd_t;
 typedef pte_t *pgtable_t;
 
 #define pgprot_val(x)	((x).pgprot)
+#define pgste_val(x)	((x).pgste)
 #define pte_val(x)	((x).pte)
 #define pmd_val(x)	((x).pmd)
 #define pud_val(x)	((x).pud)
 #define pgd_val(x)      ((x).pgd)
 
+#define __pgste(x)	((pgste_t) { (x) } )
 #define __pte(x)        ((pte_t) { (x) } )
 #define __pmd(x)        ((pmd_t) { (x) } )
+#define __pud(x)	((pud_t) { (x) } )
 #define __pgd(x)        ((pgd_t) { (x) } )
 #define __pgprot(x)     ((pgprot_t) { (x) } )
 

arch/s390/include/asm/pgalloc.h

@@ -65,10 +65,7 @@ static inline unsigned long pgd_entry_type(struct mm_struct *mm)
 #define pmd_free(mm, x)				do { } while (0)
 
 #define pgd_populate(mm, pgd, pud)		BUG()
-#define pgd_populate_kernel(mm, pgd, pud)	BUG()
-
 #define pud_populate(mm, pud, pmd)		BUG()
-#define pud_populate_kernel(mm, pud, pmd)	BUG()
 
 #else /* __s390x__ */
 
@@ -102,26 +99,14 @@ static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long vmaddr)
 }
 #define pmd_free(mm, pmd) crst_table_free(mm, (unsigned long *) pmd)
 
-static inline void pgd_populate_kernel(struct mm_struct *mm,
-				       pgd_t *pgd, pud_t *pud)
-{
-	pgd_val(*pgd) = _REGION2_ENTRY | __pa(pud);
-}
-
 static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
 {
-	pgd_populate_kernel(mm, pgd, pud);
-}
-
-static inline void pud_populate_kernel(struct mm_struct *mm,
-				       pud_t *pud, pmd_t *pmd)
-{
-	pud_val(*pud) = _REGION3_ENTRY | __pa(pmd);
+	pgd_val(*pgd) = _REGION2_ENTRY | __pa(pud);
 }
 
 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
 {
-	pud_populate_kernel(mm, pud, pmd);
+	pud_val(*pud) = _REGION3_ENTRY | __pa(pmd);
 }
 
 #endif /* __s390x__ */
@@ -134,18 +119,14 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
 }
 #define pgd_free(mm, pgd) crst_table_free(mm, (unsigned long *) pgd)
 
-static inline void pmd_populate_kernel(struct mm_struct *mm,
-				       pmd_t *pmd, pte_t *pte)
-{
-	pmd_val(*pmd) = _SEGMENT_ENTRY + __pa(pte);
-}
-
 static inline void pmd_populate(struct mm_struct *mm,
 				pmd_t *pmd, pgtable_t pte)
 {
-	pmd_populate_kernel(mm, pmd, pte);
+	pmd_val(*pmd) = _SEGMENT_ENTRY + __pa(pte);
 }
 
+#define pmd_populate_kernel(mm, pmd, pte) pmd_populate(mm, pmd, pte)
+
 #define pmd_pgtable(pmd) \
 	(pgtable_t)(pmd_val(pmd) & -sizeof(pte_t)*PTRS_PER_PTE)
 

arch/s390/mm/init.c

@@ -175,7 +175,8 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
 		pmd = pmd_offset(pud, address);
 		pte = pte_offset_kernel(pmd, address);
 		if (!enable) {
-			ptep_invalidate(&init_mm, address, pte);
+			__ptep_ipte(address, pte);
+			pte_val(*pte) = _PAGE_TYPE_EMPTY;
 			continue;
 		}
 		*pte = mk_pte_phys(address, __pgprot(_PAGE_TYPE_RW));

arch/s390/mm/pageattr.c

@@ -28,7 +28,7 @@ static void change_page_attr(unsigned long addr, int numpages,
 
 		pte = *ptep;
 		pte = set(pte);
-		ptep_invalidate(&init_mm, addr, ptep);
+		__ptep_ipte(addr, ptep);
 		*ptep = pte;
 		addr += PAGE_SIZE;
 	}

arch/s390/mm/vmem.c

@@ -95,7 +95,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
 			pu_dir = vmem_pud_alloc();
 			if (!pu_dir)
 				goto out;
-			pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
+			pgd_populate(&init_mm, pg_dir, pu_dir);
 		}
 
 		pu_dir = pud_offset(pg_dir, address);
@@ -103,7 +103,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
 			pm_dir = vmem_pmd_alloc();
 			if (!pm_dir)
 				goto out;
-			pud_populate_kernel(&init_mm, pu_dir, pm_dir);
+			pud_populate(&init_mm, pu_dir, pm_dir);
 		}
 
 		pte = mk_pte_phys(address, __pgprot(ro ? _PAGE_RO : 0));
@@ -123,7 +123,7 @@ static int vmem_add_mem(unsigned long start, unsigned long size, int ro)
 			pt_dir = vmem_pte_alloc();
 			if (!pt_dir)
 				goto out;
-			pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
+			pmd_populate(&init_mm, pm_dir, pt_dir);
 		}
 
 		pt_dir = pte_offset_kernel(pm_dir, address);
@@ -159,7 +159,7 @@ static void vmem_remove_range(unsigned long start, unsigned long size)
 			continue;
 
 		if (pmd_huge(*pm_dir)) {
-			pmd_clear_kernel(pm_dir);
+			pmd_clear(pm_dir);
 			address += HPAGE_SIZE - PAGE_SIZE;
 			continue;
 		}
@@ -192,7 +192,7 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
 			pu_dir = vmem_pud_alloc();
 			if (!pu_dir)
 				goto out;
-			pgd_populate_kernel(&init_mm, pg_dir, pu_dir);
+			pgd_populate(&init_mm, pg_dir, pu_dir);
 		}
 
 		pu_dir = pud_offset(pg_dir, address);
@@ -200,7 +200,7 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
 			pm_dir = vmem_pmd_alloc();
 			if (!pm_dir)
 				goto out;
-			pud_populate_kernel(&init_mm, pu_dir, pm_dir);
+			pud_populate(&init_mm, pu_dir, pm_dir);
 		}
 
 		pm_dir = pmd_offset(pu_dir, address);
@@ -208,7 +208,7 @@ int __meminit vmemmap_populate(struct page *start, unsigned long nr, int node)
 			pt_dir = vmem_pte_alloc();
 			if (!pt_dir)
 				goto out;
-			pmd_populate_kernel(&init_mm, pm_dir, pt_dir);
+			pmd_populate(&init_mm, pm_dir, pt_dir);
 		}
 
 		pt_dir = pte_offset_kernel(pm_dir, address);