Commit 50658d33 authored by Zwane Mwaikambo's avatar Zwane Mwaikambo Committed by Linus Torvalds

[PATCH] out-of-line locks / generic

This patch achieves out of line spinlocks by creating kernel/spinlock.c
and using the _raw_* inline locking functions.

Now, as much as this is supposed to be arch agnostic, there was still a
fair amount of rummaging about in archs, mostly for the cases where the
arch already has out of line locks and i wanted to avoid the extra call,
saving that extra call also makes lock profiling easier.  PPC32/64 was
an example of such an arch and i have added the necessary profile_pc()
function as an example. 

Size differences are with CONFIG_PREEMPT enabled since we wanted to
determine how much could be saved by moving that lot out of line too.

ppc64 = 259897 bytes:
   text    data     bss     dec     hex filename
5489808 1962724  709064 8161596  7c893c vmlinux-after
5749577 1962852  709064 8421493  808075 vmlinux-before

sparc64 = 193368 bytes:
  text    data     bss     dec     hex filename
3472037  633712  308920 4414669  435ccd vmlinux-after
3665285  633832  308920 4608037  465025 vmlinux-before

i386 = 416075 bytes
   text    data     bss     dec     hex filename
5808371  867442  326864 7002677  6ada35 vmlinux-after
6221254  870634  326864 7418752  713380 vmlinux-before

x86-64 = 282446 bytes
   text    data     bss     dec     hex filename
4598025 1450644  523632 6572301  64490d vmlinux-after
4881679 1449436  523632 6854747  68985b vmlinux-before

It has been compile tested (UP, SMP, PREEMPT) on i386, x86-64, sparc,
sparc64, ppc64, ppc32 and runtime tested on i386, x86-64 and sparc64.
Signed-off-by: default avatarZwane Mwaikambo <zwane@fsmlabs.com>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 02f42903
......@@ -19,7 +19,7 @@ static int timer_notify(struct notifier_block * self, unsigned long val, void *
{
struct pt_regs * regs = (struct pt_regs *)data;
int cpu = smp_processor_id();
unsigned long eip = instruction_pointer(regs);
unsigned long eip = profile_pc(regs);
oprofile_add_sample(eip, !user_mode(regs), 0, cpu);
return 0;
......
......@@ -77,3 +77,8 @@
VMLINUX_SYMBOL(__sched_text_start) = .; \
*(.sched.text) \
VMLINUX_SYMBOL(__sched_text_end) = .;
#define LOCK_TEXT \
VMLINUX_SYMBOL(__lock_text_start) = .; \
*(.lock.text) \
VMLINUX_SYMBOL(__lock_text_end) = .;
......@@ -19,18 +19,18 @@
/*
* Must define these before including other files, inline functions need them
*/
#define LOCK_SECTION_NAME \
".text.lock." __stringify(KBUILD_BASENAME)
#define LOCK_SECTION_NAME \
".text.lock." __stringify(KBUILD_BASENAME)
#define LOCK_SECTION_START(extra) \
".subsection 1\n\t" \
extra \
".ifndef " LOCK_SECTION_NAME "\n\t" \
LOCK_SECTION_NAME ":\n\t" \
".endif\n\t"
#define LOCK_SECTION_START(extra) \
".subsection 1\n\t" \
extra \
".ifndef " LOCK_SECTION_NAME "\n\t" \
LOCK_SECTION_NAME ":\n\t" \
".endif\n\t"
#define LOCK_SECTION_END \
".previous\n\t"
#define LOCK_SECTION_END \
".previous\n\t"
/*
* If CONFIG_SMP is set, pull in the _raw_* definitions
......@@ -38,10 +38,41 @@
#ifdef CONFIG_SMP
#include <asm/spinlock.h>
#define __lockfunc fastcall __attribute__((section(".spinlock.text")))
int __lockfunc _spin_trylock(spinlock_t *lock);
int __lockfunc _write_trylock(rwlock_t *lock);
void __lockfunc _spin_lock(spinlock_t *lock);
void __lockfunc _write_lock(rwlock_t *lock);
void __lockfunc _spin_lock(spinlock_t *lock);
void __lockfunc _read_lock(rwlock_t *lock);
void __lockfunc _spin_unlock(spinlock_t *lock);
void __lockfunc _write_unlock(rwlock_t *lock);
void __lockfunc _read_unlock(rwlock_t *lock);
unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock);
unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock);
unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock);
void __lockfunc _spin_lock_irq(spinlock_t *lock);
void __lockfunc _spin_lock_bh(spinlock_t *lock);
void __lockfunc _read_lock_irq(rwlock_t *lock);
void __lockfunc _read_lock_bh(rwlock_t *lock);
void __lockfunc _write_lock_irq(rwlock_t *lock);
void __lockfunc _write_lock_bh(rwlock_t *lock);
void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags);
void __lockfunc _spin_unlock_irq(spinlock_t *lock);
void __lockfunc _spin_unlock_bh(spinlock_t *lock);
void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags);
void __lockfunc _read_unlock_irq(rwlock_t *lock);
void __lockfunc _read_unlock_bh(rwlock_t *lock);
void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags);
void __lockfunc _write_unlock_irq(rwlock_t *lock);
void __lockfunc _write_unlock_bh(rwlock_t *lock);
int __lockfunc _spin_trylock_bh(spinlock_t *lock);
extern unsigned long __lock_text_start;
extern unsigned long __lock_text_end;
#else
#define _raw_spin_lock_flags(lock, flags) _raw_spin_lock(lock)
#if !defined(CONFIG_PREEMPT) && !defined(CONFIG_DEBUG_SPINLOCK)
# define atomic_dec_and_lock(atomic,lock) atomic_dec_and_test(atomic)
# define ATOMIC_DEC_AND_LOCK
......@@ -162,9 +193,9 @@ typedef struct {
#define spin_lock_init(lock) do { (void)(lock); } while(0)
#define _raw_spin_lock(lock) do { (void)(lock); } while(0)
#define spin_is_locked(lock) ((void)(lock), 0)
#define _raw_spin_trylock(lock) ((void)(lock), 1)
#define spin_unlock_wait(lock) do { (void)(lock); } while(0)
#define _raw_spin_unlock(lock) do { (void)(lock); } while(0)
#define _raw_spin_trylock(lock) (((void)(lock), 1))
#define spin_unlock_wait(lock) (void)(lock);
#define _raw_spin_unlock(lock) do { (void)(lock); } while(0)
#endif /* CONFIG_DEBUG_SPINLOCK */
/* RW spinlocks: No debug version */
......@@ -184,141 +215,116 @@ typedef struct {
#define _raw_write_unlock(lock) do { (void)(lock); } while(0)
#define _raw_write_trylock(lock) ({ (void)(lock); (1); })
#endif /* !SMP */
/*
* Define the various spin_lock and rw_lock methods. Note we define these
* regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various
* methods are defined as nops in the case they are not required.
*/
#define spin_trylock(lock) ({preempt_disable(); _raw_spin_trylock(lock) ? \
#define _spin_trylock(lock) ({preempt_disable(); _raw_spin_trylock(lock) ? \
1 : ({preempt_enable(); 0;});})
#define write_trylock(lock) ({preempt_disable();_raw_write_trylock(lock) ? \
#define _write_trylock(lock) ({preempt_disable(); _raw_write_trylock(lock) ? \
1 : ({preempt_enable(); 0;});})
/* Where's read_trylock? */
#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
void __preempt_spin_lock(spinlock_t *lock);
void __preempt_write_lock(rwlock_t *lock);
#define _spin_trylock_bh(lock) ({preempt_disable(); local_bh_disable(); \
_raw_spin_trylock(lock) ? \
1 : ({preempt_enable(); local_bh_enable(); 0;});})
#define spin_lock(lock) \
do { \
preempt_disable(); \
if (unlikely(!_raw_spin_trylock(lock))) \
__preempt_spin_lock(lock); \
} while (0)
#define write_lock(lock) \
do { \
preempt_disable(); \
if (unlikely(!_raw_write_trylock(lock))) \
__preempt_write_lock(lock); \
} while (0)
#else
#define spin_lock(lock) \
#define _spin_lock(lock) \
do { \
preempt_disable(); \
_raw_spin_lock(lock); \
} while(0)
#define write_lock(lock) \
#define _write_lock(lock) \
do { \
preempt_disable(); \
_raw_write_lock(lock); \
} while(0)
#endif
#define read_lock(lock) \
#define _read_lock(lock) \
do { \
preempt_disable(); \
_raw_read_lock(lock); \
} while(0)
#define spin_unlock(lock) \
#define _spin_unlock(lock) \
do { \
_raw_spin_unlock(lock); \
preempt_enable(); \
} while (0)
#define write_unlock(lock) \
#define _write_unlock(lock) \
do { \
_raw_write_unlock(lock); \
preempt_enable(); \
} while(0)
#define read_unlock(lock) \
#define _read_unlock(lock) \
do { \
_raw_read_unlock(lock); \
preempt_enable(); \
} while(0)
#define spin_lock_irqsave(lock, flags) \
do { \
#define _spin_lock_irqsave(lock, flags) \
do { \
local_irq_save(flags); \
preempt_disable(); \
_raw_spin_lock_flags(lock, flags); \
_raw_spin_lock(lock); \
} while (0)
#define spin_lock_irq(lock) \
#define _spin_lock_irq(lock) \
do { \
local_irq_disable(); \
preempt_disable(); \
_raw_spin_lock(lock); \
} while (0)
#define spin_lock_bh(lock) \
#define _spin_lock_bh(lock) \
do { \
local_bh_disable(); \
preempt_disable(); \
_raw_spin_lock(lock); \
} while (0)
#define read_lock_irqsave(lock, flags) \
do { \
#define _read_lock_irqsave(lock, flags) \
do { \
local_irq_save(flags); \
preempt_disable(); \
_raw_read_lock(lock); \
} while (0)
#define read_lock_irq(lock) \
#define _read_lock_irq(lock) \
do { \
local_irq_disable(); \
preempt_disable(); \
_raw_read_lock(lock); \
} while (0)
#define read_lock_bh(lock) \
#define _read_lock_bh(lock) \
do { \
local_bh_disable(); \
preempt_disable(); \
_raw_read_lock(lock); \
} while (0)
#define write_lock_irqsave(lock, flags) \
do { \
#define _write_lock_irqsave(lock, flags) \
do { \
local_irq_save(flags); \
preempt_disable(); \
_raw_write_lock(lock); \
} while (0)
#define write_lock_irq(lock) \
#define _write_lock_irq(lock) \
do { \
local_irq_disable(); \
preempt_disable(); \
_raw_write_lock(lock); \
} while (0)
#define write_lock_bh(lock) \
#define _write_lock_bh(lock) \
do { \
local_bh_disable(); \
preempt_disable(); \
_raw_write_lock(lock); \
} while (0)
#define spin_unlock_irqrestore(lock, flags) \
#define _spin_unlock_irqrestore(lock, flags) \
do { \
_raw_spin_unlock(lock); \
local_irq_restore(flags); \
......@@ -331,65 +337,128 @@ do { \
local_irq_restore(flags); \
} while (0)
#define spin_unlock_irq(lock) \
#define _spin_unlock_irq(lock) \
do { \
_raw_spin_unlock(lock); \
local_irq_enable(); \
preempt_enable(); \
} while (0)
#define spin_unlock_bh(lock) \
#define _spin_unlock_bh(lock) \
do { \
_raw_spin_unlock(lock); \
preempt_enable(); \
local_bh_enable(); \
} while (0)
#define read_unlock_irqrestore(lock, flags) \
#define _write_unlock_bh(lock) \
do { \
_raw_read_unlock(lock); \
local_irq_restore(flags); \
_raw_write_unlock(lock); \
preempt_enable(); \
local_bh_enable(); \
} while (0)
#define read_unlock_irq(lock) \
#define _read_unlock_irqrestore(lock, flags) \
do { \
_raw_read_unlock(lock); \
local_irq_enable(); \
local_irq_restore(flags); \
preempt_enable(); \
} while (0)
#define read_unlock_bh(lock) \
#define _write_unlock_irqrestore(lock, flags) \
do { \
_raw_read_unlock(lock); \
_raw_write_unlock(lock); \
local_irq_restore(flags); \
preempt_enable(); \
local_bh_enable(); \
} while (0)
#define write_unlock_irqrestore(lock, flags) \
#define _read_unlock_irq(lock) \
do { \
_raw_write_unlock(lock); \
local_irq_restore(flags); \
preempt_enable(); \
_raw_read_unlock(lock); \
local_irq_enable(); \
preempt_enable(); \
} while (0)
#define write_unlock_irq(lock) \
#define _read_unlock_bh(lock) \
do { \
_raw_write_unlock(lock); \
local_irq_enable(); \
preempt_enable(); \
_raw_read_unlock(lock); \
local_bh_enable(); \
preempt_enable(); \
} while (0)
#define write_unlock_bh(lock) \
#define _write_unlock_irq(lock) \
do { \
_raw_write_unlock(lock); \
preempt_enable(); \
local_bh_enable(); \
_raw_write_unlock(lock); \
local_irq_enable(); \
preempt_enable(); \
} while (0)
#define spin_trylock_bh(lock) ({ local_bh_disable(); preempt_disable(); \
_raw_spin_trylock(lock) ? 1 : \
({preempt_enable(); local_bh_enable(); 0;});})
#endif /* !SMP */
/*
* Define the various spin_lock and rw_lock methods. Note we define these
* regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various
* methods are defined as nops in the case they are not required.
*/
#define spin_trylock(lock) _spin_trylock(lock)
#define write_trylock(lock) _write_trylock(lock)
/* Where's read_trylock? */
#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
void __preempt_spin_lock(spinlock_t *lock);
void __preempt_write_lock(rwlock_t *lock);
#endif
#define spin_lock(lock) _spin_lock(lock)
#define write_lock(lock) _write_lock(lock)
#define read_lock(lock) _read_lock(lock)
#define spin_unlock(lock) _spin_unlock(lock)
#define write_unlock(lock) _write_unlock(lock)
#define read_unlock(lock) _read_unlock(lock)
#ifdef CONFIG_SMP
#define spin_lock_irqsave(lock, flags) flags = _spin_lock_irqsave(lock)
#define read_lock_irqsave(lock, flags) flags = _read_lock_irqsave(lock)
#define write_lock_irqsave(lock, flags) flags = _write_lock_irqsave(lock)
#else
#define spin_lock_irqsave(lock, flags) _spin_lock_irqsave(lock, flags)
#define read_lock_irqsave(lock, flags) _read_lock_irqsave(lock, flags)
#define write_lock_irqsave(lock, flags) _write_lock_irqsave(lock, flags)
#endif
#define spin_lock_irq(lock) _spin_lock_irq(lock)
#define spin_lock_bh(lock) _spin_lock_bh(lock)
#define read_lock_irq(lock) _read_lock_irq(lock)
#define read_lock_bh(lock) _read_lock_bh(lock)
#define write_lock_irq(lock) _write_lock_irq(lock)
#define write_lock_bh(lock) _write_lock_bh(lock)
#define spin_unlock_irqrestore(lock, flags) _spin_unlock_irqrestore(lock, flags)
#define spin_unlock_irq(lock) _spin_unlock_irq(lock)
#define spin_unlock_bh(lock) _spin_unlock_bh(lock)
#define read_unlock_irqrestore(lock, flags) _read_unlock_irqrestore(lock, flags)
#define read_unlock_irq(lock) _read_unlock_irq(lock)
#define read_unlock_bh(lock) _read_unlock_bh(lock)
#define write_unlock_irqrestore(lock, flags) _write_unlock_irqrestore(lock, flags)
#define write_unlock_irq(lock) _write_unlock_irq(lock)
#define write_unlock_bh(lock) _write_unlock_bh(lock)
#define spin_trylock_bh(lock) _spin_trylock_bh(lock)
#ifdef CONFIG_LOCKMETER
extern void _metered_spin_lock (spinlock_t *lock);
extern void _metered_spin_unlock (spinlock_t *lock);
extern int _metered_spin_trylock(spinlock_t *lock);
extern void _metered_read_lock (rwlock_t *lock);
extern void _metered_read_unlock (rwlock_t *lock);
extern void _metered_write_lock (rwlock_t *lock);
extern void _metered_write_unlock (rwlock_t *lock);
extern int _metered_write_trylock(rwlock_t *lock);
#endif
/* "lock on reference count zero" */
#ifndef ATOMIC_DEC_AND_LOCK
......
......@@ -11,7 +11,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \
obj-$(CONFIG_FUTEX) += futex.o
obj-$(CONFIG_GENERIC_ISA_DMA) += dma.o
obj-$(CONFIG_SMP) += cpu.o
obj-$(CONFIG_SMP) += cpu.o spinlock.o
obj-$(CONFIG_UID16) += uid16.o
obj-$(CONFIG_MODULES) += module.o
obj-$(CONFIG_KALLSYMS) += kallsyms.o
......
/*
* Copyright (2004) Linus Torvalds
*
* Author: Zwane Mwaikambo <zwane@fsmlabs.com>
*/
#include <linux/config.h>
#include <linux/linkage.h>
#include <linux/preempt.h>
#include <linux/spinlock.h>
#include <linux/interrupt.h>
#include <linux/module.h>
int __lockfunc _spin_trylock(spinlock_t *lock)
{
preempt_disable();
if (_raw_spin_trylock(lock))
return 1;
preempt_enable();
return 0;
}
EXPORT_SYMBOL(_spin_trylock);
int __lockfunc _write_trylock(rwlock_t *lock)
{
preempt_disable();
if (_raw_write_trylock(lock))
return 1;
preempt_enable();
return 0;
}
EXPORT_SYMBOL(_write_trylock);
#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
void __lockfunc _spin_lock(spinlock_t *lock)
{
preempt_disable();
if (unlikely(!_raw_spin_trylock(lock)))
__preempt_spin_lock(lock);
}
void __lockfunc _write_lock(rwlock_t *lock)
{
preempt_disable();
if (unlikely(!_raw_write_trylock(lock)))
__preempt_write_lock(lock);
}
#else
void __lockfunc _spin_lock(spinlock_t *lock)
{
preempt_disable();
_raw_spin_lock(lock);
}
void __lockfunc _write_lock(rwlock_t *lock)
{
preempt_disable();
_raw_write_lock(lock);
}
#endif
EXPORT_SYMBOL(_spin_lock);
EXPORT_SYMBOL(_write_lock);
void __lockfunc _read_lock(rwlock_t *lock)
{
preempt_disable();
_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock);
void __lockfunc _spin_unlock(spinlock_t *lock)
{
_raw_spin_unlock(lock);
preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock);
void __lockfunc _write_unlock(rwlock_t *lock)
{
_raw_write_unlock(lock);
preempt_enable();
}
EXPORT_SYMBOL(_write_unlock);
void __lockfunc _read_unlock(rwlock_t *lock)
{
_raw_read_unlock(lock);
preempt_enable();
}
EXPORT_SYMBOL(_read_unlock);
unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
{
unsigned long flags;
local_irq_save(flags);
preempt_disable();
_raw_spin_lock_flags(lock, flags);
return flags;
}
EXPORT_SYMBOL(_spin_lock_irqsave);
void __lockfunc _spin_lock_irq(spinlock_t *lock)
{
local_irq_disable();
preempt_disable();
_raw_spin_lock(lock);
}
EXPORT_SYMBOL(_spin_lock_irq);
void __lockfunc _spin_lock_bh(spinlock_t *lock)
{
local_bh_disable();
preempt_disable();
_raw_spin_lock(lock);
}
EXPORT_SYMBOL(_spin_lock_bh);
unsigned long __lockfunc _read_lock_irqsave(rwlock_t *lock)
{
unsigned long flags;
local_irq_save(flags);
preempt_disable();
_raw_read_lock(lock);
return flags;
}
EXPORT_SYMBOL(_read_lock_irqsave);
void __lockfunc _read_lock_irq(rwlock_t *lock)
{
local_irq_disable();
preempt_disable();
_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock_irq);
void __lockfunc _read_lock_bh(rwlock_t *lock)
{
local_bh_disable();
preempt_disable();
_raw_read_lock(lock);
}
EXPORT_SYMBOL(_read_lock_bh);
unsigned long __lockfunc _write_lock_irqsave(rwlock_t *lock)
{
unsigned long flags;
local_irq_save(flags);
preempt_disable();
_raw_write_lock(lock);
return flags;
}
EXPORT_SYMBOL(_write_lock_irqsave);
void __lockfunc _write_lock_irq(rwlock_t *lock)
{
local_irq_disable();
preempt_disable();
_raw_write_lock(lock);
}
EXPORT_SYMBOL(_write_lock_irq);
void __lockfunc _write_lock_bh(rwlock_t *lock)
{
local_bh_disable();
preempt_disable();
_raw_write_lock(lock);
}
EXPORT_SYMBOL(_write_lock_bh);
void __lockfunc _spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags)
{
_raw_spin_unlock(lock);
local_irq_restore(flags);
preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irqrestore);
void __lockfunc _spin_unlock_irq(spinlock_t *lock)
{
_raw_spin_unlock(lock);
local_irq_enable();
preempt_enable();
}
EXPORT_SYMBOL(_spin_unlock_irq);
void __lockfunc _spin_unlock_bh(spinlock_t *lock)
{
_raw_spin_unlock(lock);
preempt_enable();
local_bh_enable();
}
EXPORT_SYMBOL(_spin_unlock_bh);
void __lockfunc _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
_raw_read_unlock(lock);
local_irq_restore(flags);
preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irqrestore);
void __lockfunc _read_unlock_irq(rwlock_t *lock)
{
_raw_read_unlock(lock);
local_irq_enable();
preempt_enable();
}
EXPORT_SYMBOL(_read_unlock_irq);
void __lockfunc _read_unlock_bh(rwlock_t *lock)
{
_raw_read_unlock(lock);
preempt_enable();
local_bh_enable();
}
EXPORT_SYMBOL(_read_unlock_bh);
void __lockfunc _write_unlock_irqrestore(rwlock_t *lock, unsigned long flags)
{
_raw_write_unlock(lock);
local_irq_restore(flags);
preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irqrestore);
void __lockfunc _write_unlock_irq(rwlock_t *lock)
{
_raw_write_unlock(lock);
local_irq_enable();
preempt_enable();
}
EXPORT_SYMBOL(_write_unlock_irq);
void __lockfunc _write_unlock_bh(rwlock_t *lock)
{
_raw_write_unlock(lock);
preempt_enable();
local_bh_enable();
}
EXPORT_SYMBOL(_write_unlock_bh);
int __lockfunc _spin_trylock_bh(spinlock_t *lock)
{
local_bh_disable();
preempt_disable();
if (_raw_spin_trylock(lock))
return 1;
preempt_enable();
local_bh_enable();
return 0;
}
EXPORT_SYMBOL(_spin_trylock_bh);
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment