[PATCH] Allow i386 to reenable interrupts on lock contention

From: Zwane Mwaikambo <zwane@linuxpower.ca>

Following up on Keith's code, I adapted the i386 code to allow enabling
interrupts during contested locks depending on previous interrupt
enable status. Obviously there will be a text increase (only for non
CONFIG_SPINLINE case), although it doesn't seem so bad, there will be an
increased exit latency when we attempt a lock acquisition after spinning
due to the extra instructions. How much this will affect performance I'm
not sure yet as I haven't had time to micro bench.

   text    data     bss     dec     hex filename
2628024  921731       0 3549755  362a3b vmlinux-after
2621369  921731       0 3543100  36103c vmlinux-before
2618313  919222       0 3537535  35fa7f vmlinux-spinline

The code has been stress tested on a 16x NUMAQ (courtesy OSDL).
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

include/asm-i386/spinlock.h

@@ -42,7 +42,6 @@ typedef struct {
 
 #define spin_is_locked(x)	(*(volatile signed char *)(&(x)->lock) <= 0)
 #define spin_unlock_wait(x)	do { barrier(); } while(spin_is_locked(x))
-#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
 
 #define spin_lock_string \
 	"\n1:\t" \
@@ -56,6 +55,23 @@ typedef struct {
 	"jmp 1b\n" \
 	LOCK_SECTION_END
 
+#define spin_lock_string_flags \
+	"\n1:\t" \
+	"lock ; decb %0\n\t" \
+	"js 2f\n\t" \
+	LOCK_SECTION_START("") \
+	"2:\t" \
+	"testl $0x200, %1\n\t" \
+	"jz 3f\n\t" \
+	"sti\n\t" \
+	"3:\t" \
+	"rep;nop\n\t" \
+	"cmpb $0, %0\n\t" \
+	"jle 3b\n\t" \
+	"cli\n\t" \
+	"jmp 1b\n" \
+	LOCK_SECTION_END
+
 /*
  * This works. Despite all the confusion.
  * (except on PPro SMP or if we are using OOSTORE)
@@ -126,6 +142,20 @@ static inline void _raw_spin_lock(spinlock_t *lock)
 		:"=m" (lock->lock) : : "memory");
 }
 
+static inline void _raw_spin_lock_flags (spinlock_t *lock, unsigned long flags)
+{
+#ifdef CONFIG_DEBUG_SPINLOCK
+	__label__ here;
+here:
+	if (unlikely(lock->magic != SPINLOCK_MAGIC)) {
+		printk("eip: %p\n", &&here);
+		BUG();
+	}
+#endif
+	__asm__ __volatile__(
+		spin_lock_string_flags
+		:"=m" (lock->lock) : "r" (flags) : "memory");
+}
 
 /*
  * Read-write spinlocks, allowing multiple readers