[PATCH] ppc64: hash table races fixes

This fixes some possible rare issues with the hash code beeing called
with interrupts enabled from update_mmu_cache, and fixes some races in
the iSeries low level htab code by adding an array of spinlocks

This is actually an old patch already present in SLES kernel and that
got "missed" somewhat in the main tree, adapted to recent changes.
Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

arch/ppc64/kernel/iSeries_htab.c

@@ -16,17 +16,26 @@
 #include <asm/mmu_context.h>
 #include <asm/iSeries/HvCallHpt.h>
 #include <asm/abs_addr.h>
-
-#if 0
 #include <linux/spinlock.h>
-#include <linux/bitops.h>
-#include <linux/threads.h>
-#include <linux/smp.h>
 
-#include <asm/tlbflush.h>
-#include <asm/tlb.h>
-#include <asm/cputable.h>
-#endif
+static spinlock_t iSeries_hlocks[64] __cacheline_aligned_in_smp = { [0 ... 63] = SPIN_LOCK_UNLOCKED};
+
+/*
+ * Very primitive algorithm for picking up a lock
+ */
+static inline void iSeries_hlock(unsigned long slot)
+{
+	if (slot & 0x8)
+		slot = ~slot;
+	spin_lock(&iSeries_hlocks[(slot >> 4) & 0x3f]);
+}
+
+static inline void iSeries_hunlock(unsigned long slot)
+{
+	if (slot & 0x8)
+		slot = ~slot;
+	spin_unlock(&iSeries_hlocks[(slot >> 4) & 0x3f]);
+}
 
 static long iSeries_hpte_insert(unsigned long hpte_group, unsigned long va,
 			 unsigned long prpn, int secondary,
@@ -44,12 +53,15 @@ static long iSeries_hpte_insert(unsigned long hpte_group, unsigned long va,
 	if (secondary)
 		return -1;
 
+	iSeries_hlock(hpte_group);
+
 	slot = HvCallHpt_findValid(&lhpte, va >> PAGE_SHIFT);
-	if (lhpte.dw0.dw0.v)
-		panic("select_hpte_slot found entry already valid\n");
+	BUG_ON(lhpte.dw0.dw0.v);
 
-	if (slot == -1)	/* No available entry found in either group */
+	if (slot == -1)	{ /* No available entry found in either group */
+		iSeries_hunlock(hpte_group);
 		return -1;
+	}
 
 	if (slot < 0) {		/* MSB set means secondary group */
 		secondary = 1;
@@ -69,6 +81,8 @@ static long iSeries_hpte_insert(unsigned long hpte_group, unsigned long va,
 	/* Now fill in the actual HPTE */
 	HvCallHpt_addValidate(slot, secondary, &lhpte);
 
+	iSeries_hunlock(hpte_group);
+
 	return (secondary << 3) | (slot & 7);
 }
 
@@ -92,6 +106,8 @@ static long iSeries_hpte_remove(unsigned long hpte_group)
 	/* Pick a random slot to start at */
 	slot_offset = mftb() & 0x7;
 
+	iSeries_hlock(hpte_group);
+
 	for (i = 0; i < HPTES_PER_GROUP; i++) {
 		lhpte.dw0.dword0 = 
 			iSeries_hpte_getword0(hpte_group + slot_offset);
@@ -99,6 +115,7 @@ static long iSeries_hpte_remove(unsigned long hpte_group)
 		if (!lhpte.dw0.dw0.bolted) {
 			HvCallHpt_invalidateSetSwBitsGet(hpte_group + 
 							 slot_offset, 0, 0);
+			iSeries_hunlock(hpte_group);
 			return i;
 		}
 
@@ -106,6 +123,8 @@ static long iSeries_hpte_remove(unsigned long hpte_group)
 		slot_offset &= 0x7;
 	}
 
+	iSeries_hunlock(hpte_group);
+
 	return -1;
 }
 
@@ -121,11 +140,16 @@ static long iSeries_hpte_updatepp(unsigned long slot, unsigned long newpp,
 	HPTE hpte;
 	unsigned long avpn = va >> 23;
 
+	iSeries_hlock(slot);
+
 	HvCallHpt_get(&hpte, slot);
 	if ((hpte.dw0.dw0.avpn == avpn) && (hpte.dw0.dw0.v)) {
 		HvCallHpt_setPp(slot, (newpp & 0x3) | ((newpp & 0x4) << 1));
+		iSeries_hunlock(slot);
 		return 0;
 	}
+	iSeries_hunlock(slot);
+
 	return -1;
 }
 
@@ -186,11 +210,20 @@ static void iSeries_hpte_invalidate(unsigned long slot, unsigned long va,
 {
 	HPTE lhpte;
 	unsigned long avpn = va >> 23;
+	unsigned long flags;
+
+	local_irq_save(flags);
+
+	iSeries_hlock(slot);
 
 	lhpte.dw0.dword0 = iSeries_hpte_getword0(slot);
 	
 	if ((lhpte.dw0.dw0.avpn == avpn) && lhpte.dw0.dw0.v)
 		HvCallHpt_invalidateSetSwBitsGet(slot, 0, 0);
+
+	iSeries_hunlock(slot);
+
+	local_irq_restore(flags);
 }
 
 void hpte_init_iSeries(void)

arch/ppc64/kernel/pSeries_htab.c

@@ -198,7 +198,6 @@ static long pSeries_hpte_updatepp(unsigned long slot, unsigned long newpp,
 	HPTE *hptep = htab_data.htab + slot;
 	Hpte_dword0 dw0;
 	unsigned long avpn = va >> 23;
-	unsigned long flags;
 	int ret = 0;
 
 	if (large)
@@ -222,10 +221,10 @@ static long pSeries_hpte_updatepp(unsigned long slot, unsigned long newpp,
 		tlbiel(va);
 	} else {
 		if (!(cur_cpu_spec->cpu_features & CPU_FTR_LOCKLESS_TLBIE))
-			spin_lock_irqsave(&pSeries_tlbie_lock, flags);
+			spin_lock(&pSeries_tlbie_lock);
 		tlbie(va, large);
 		if (!(cur_cpu_spec->cpu_features & CPU_FTR_LOCKLESS_TLBIE))
-			spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
+			spin_unlock(&pSeries_tlbie_lock);
 	}
 
 	return ret;
@@ -275,6 +274,7 @@ static void pSeries_hpte_invalidate(unsigned long slot, unsigned long va,
 	if (large)
 		avpn &= ~0x1UL;
 
+	local_irq_save(flags);
 	pSeries_lock_hpte(hptep);
 
 	dw0 = hptep->dw0.dw0;
@@ -292,11 +292,12 @@ static void pSeries_hpte_invalidate(unsigned long slot, unsigned long va,
 		tlbiel(va);
 	} else {
 		if (!(cur_cpu_spec->cpu_features & CPU_FTR_LOCKLESS_TLBIE))
-			spin_lock_irqsave(&pSeries_tlbie_lock, flags);
+			spin_lock(&pSeries_tlbie_lock);
 		tlbie(va, large);
 		if (!(cur_cpu_spec->cpu_features & CPU_FTR_LOCKLESS_TLBIE))
-			spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
+			spin_unlock(&pSeries_tlbie_lock);
 	}
+	local_irq_restore(flags);
 }
 
 static void pSeries_flush_hash_range(unsigned long context,
@@ -311,6 +312,8 @@ static void pSeries_flush_hash_range(unsigned long context,
 	/* XXX fix for large ptes */
 	unsigned long large = 0;
 
+	local_irq_save(flags);
+
 	j = 0;
 	for (i = 0; i < number; i++) {
 		if ((batch->addr[i] >= USER_START) &&
@@ -363,7 +366,7 @@ static void pSeries_flush_hash_range(unsigned long context,
 	} else {
 		/* XXX double check that it is safe to take this late */
 		if (!(cur_cpu_spec->cpu_features & CPU_FTR_LOCKLESS_TLBIE))
-			spin_lock_irqsave(&pSeries_tlbie_lock, flags);
+			spin_lock(&pSeries_tlbie_lock);
 
 		asm volatile("ptesync":::"memory");
 
@@ -373,8 +376,10 @@ static void pSeries_flush_hash_range(unsigned long context,
 		asm volatile("eieio; tlbsync; ptesync":::"memory");
 
 		if (!(cur_cpu_spec->cpu_features & CPU_FTR_LOCKLESS_TLBIE))
-			spin_unlock_irqrestore(&pSeries_tlbie_lock, flags);
+			spin_unlock(&pSeries_tlbie_lock);
 	}
+
+	local_irq_restore(flags);
 }
 
 void hpte_init_pSeries(void)

arch/ppc64/mm/hash_utils.c

@@ -251,7 +251,6 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
 	struct mm_struct *mm;
 	pte_t *ptep;
 	int ret;
-	int cpu;
 	int user_region = 0;
 	int local = 0;
 	cpumask_t tmp;
@@ -303,8 +302,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
 	if (pgdir == NULL)
 		return 1;
 
-	cpu = get_cpu();
-	tmp = cpumask_of_cpu(cpu);
+	tmp = cpumask_of_cpu(smp_processor_id());
 	if (user_region && cpus_equal(mm->cpu_vm_mask, tmp))
 		local = 1;
 
@@ -313,13 +311,10 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap)
 		ret = hash_huge_page(mm, access, ea, vsid, local);
 	else {
 		ptep = find_linux_pte(pgdir, ea);
-		if (ptep == NULL) {
-			put_cpu();
+		if (ptep == NULL)
 			return 1;
-		}
 		ret = __hash_page(ea, access, vsid, ptep, trap, local);
 	}
-	put_cpu();
 
 	return ret;
 }

arch/ppc64/mm/init.c

@@ -765,8 +765,8 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long ea,
 	void *pgdir;
 	pte_t *ptep;
 	int local = 0;
-	int cpu;
 	cpumask_t tmp;
+	unsigned long flags;
 
 	/* handle i-cache coherency */
 	if (!(cur_cpu_spec->cpu_features & CPU_FTR_COHERENT_ICACHE) &&
@@ -796,14 +796,14 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long ea,
 
 	vsid = get_vsid(vma->vm_mm->context.id, ea);
 
-	cpu = get_cpu();
-	tmp = cpumask_of_cpu(cpu);
+	local_irq_save(flags);
+	tmp = cpumask_of_cpu(smp_processor_id());
 	if (cpus_equal(vma->vm_mm->cpu_vm_mask, tmp))
 		local = 1;
 
 	__hash_page(ea, pte_val(pte) & (_PAGE_USER|_PAGE_RW), vsid, ptep,
 		    0x300, local);
-	put_cpu();
+	local_irq_restore(flags);
 }
 
 void * reserve_phb_iospace(unsigned long size)