Commit de30a2b3 authored by Ingo Molnar's avatar Ingo Molnar Committed by Linus Torvalds

[PATCH] lockdep: irqtrace subsystem, core

Accurate hard-IRQ-flags and softirq-flags state tracing.

This allows us to attach extra functionality to IRQ flags on/off
events (such as trace-on/off).
Signed-off-by: default avatarIngo Molnar <mingo@elte.hu>
Signed-off-by: default avatarArjan van de Ven <arjan@linux.intel.com>
Signed-off-by: default avatarAndrew Morton <akpm@osdl.org>
Signed-off-by: default avatarLinus Torvalds <torvalds@osdl.org>
parent 5bdc9b44
...@@ -429,7 +429,7 @@ void do_softirq(void) ...@@ -429,7 +429,7 @@ void do_softirq(void)
local_bh_disable(); local_bh_disable();
do_softirq_onstack(); do_softirq_onstack();
account_system_vtime(current); account_system_vtime(current);
__local_bh_enable(); _local_bh_enable();
} }
local_irq_restore(flags); local_irq_restore(flags);
......
/*
* include/asm-powerpc/irqflags.h
*
* IRQ flags handling
*
* This file gets included from lowlevel asm headers too, to provide
* wrapped versions of the local_irq_*() APIs, based on the
* raw_local_irq_*() macros from the lowlevel headers.
*/
#ifndef _ASM_IRQFLAGS_H
#define _ASM_IRQFLAGS_H
/*
* Get definitions for raw_local_save_flags(x), etc.
*/
#include <asm-powerpc/hw_irq.h>
/*
* Do the CPU's IRQ-state tracing from assembly code. We call a
* C function, so save all the C-clobbered registers:
*/
#ifdef CONFIG_TRACE_IRQFLAGS
#error No support on PowerPC yet for CONFIG_TRACE_IRQFLAGS
#else
# define TRACE_IRQS_ON
# define TRACE_IRQS_OFF
#endif
#endif
...@@ -86,9 +86,6 @@ extern void synchronize_irq(unsigned int irq); ...@@ -86,9 +86,6 @@ extern void synchronize_irq(unsigned int irq);
# define synchronize_irq(irq) barrier() # define synchronize_irq(irq) barrier()
#endif #endif
#define nmi_enter() irq_enter()
#define nmi_exit() sub_preempt_count(HARDIRQ_OFFSET)
struct task_struct; struct task_struct;
#ifndef CONFIG_VIRT_CPU_ACCOUNTING #ifndef CONFIG_VIRT_CPU_ACCOUNTING
...@@ -97,12 +94,35 @@ static inline void account_system_vtime(struct task_struct *tsk) ...@@ -97,12 +94,35 @@ static inline void account_system_vtime(struct task_struct *tsk)
} }
#endif #endif
/*
* It is safe to do non-atomic ops on ->hardirq_context,
* because NMI handlers may not preempt and the ops are
* always balanced, so the interrupted value of ->hardirq_context
* will always be restored.
*/
#define irq_enter() \ #define irq_enter() \
do { \ do { \
account_system_vtime(current); \ account_system_vtime(current); \
add_preempt_count(HARDIRQ_OFFSET); \ add_preempt_count(HARDIRQ_OFFSET); \
trace_hardirq_enter(); \
} while (0)
/*
* Exit irq context without processing softirqs:
*/
#define __irq_exit() \
do { \
trace_hardirq_exit(); \
account_system_vtime(current); \
sub_preempt_count(HARDIRQ_OFFSET); \
} while (0) } while (0)
/*
* Exit irq context and process softirqs if needed:
*/
extern void irq_exit(void); extern void irq_exit(void);
#define nmi_enter() irq_enter()
#define nmi_exit() __irq_exit()
#endif /* LINUX_HARDIRQ_H */ #endif /* LINUX_HARDIRQ_H */
...@@ -3,6 +3,7 @@ ...@@ -3,6 +3,7 @@
#include <linux/file.h> #include <linux/file.h>
#include <linux/rcupdate.h> #include <linux/rcupdate.h>
#include <linux/irqflags.h>
#define INIT_FDTABLE \ #define INIT_FDTABLE \
{ \ { \
...@@ -124,6 +125,7 @@ extern struct group_info init_groups; ...@@ -124,6 +125,7 @@ extern struct group_info init_groups;
.cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \ .cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \
.fs_excl = ATOMIC_INIT(0), \ .fs_excl = ATOMIC_INIT(0), \
.pi_lock = SPIN_LOCK_UNLOCKED, \ .pi_lock = SPIN_LOCK_UNLOCKED, \
INIT_TRACE_IRQFLAGS \
} }
......
...@@ -10,6 +10,7 @@ ...@@ -10,6 +10,7 @@
#include <linux/irqreturn.h> #include <linux/irqreturn.h>
#include <linux/hardirq.h> #include <linux/hardirq.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/irqflags.h>
#include <asm/atomic.h> #include <asm/atomic.h>
#include <asm/ptrace.h> #include <asm/ptrace.h>
#include <asm/system.h> #include <asm/system.h>
...@@ -199,13 +200,11 @@ static inline void __deprecated save_and_cli(unsigned long *x) ...@@ -199,13 +200,11 @@ static inline void __deprecated save_and_cli(unsigned long *x)
#define save_and_cli(x) save_and_cli(&x) #define save_and_cli(x) save_and_cli(&x)
#endif /* CONFIG_SMP */ #endif /* CONFIG_SMP */
/* SoftIRQ primitives. */ extern void local_bh_disable(void);
#define local_bh_disable() \ extern void __local_bh_enable(void);
do { add_preempt_count(SOFTIRQ_OFFSET); barrier(); } while (0) extern void _local_bh_enable(void);
#define __local_bh_enable() \
do { barrier(); sub_preempt_count(SOFTIRQ_OFFSET); } while (0)
extern void local_bh_enable(void); extern void local_bh_enable(void);
extern void local_bh_enable_ip(unsigned long ip);
/* PLEASE, avoid to allocate new softirqs, if you need not _really_ high /* PLEASE, avoid to allocate new softirqs, if you need not _really_ high
frequency threaded job scheduling. For almost all the purposes frequency threaded job scheduling. For almost all the purposes
......
/*
* include/linux/irqflags.h
*
* IRQ flags tracing: follow the state of the hardirq and softirq flags and
* provide callbacks for transitions between ON and OFF states.
*
* This file gets included from lowlevel asm headers too, to provide
* wrapped versions of the local_irq_*() APIs, based on the
* raw_local_irq_*() macros from the lowlevel headers.
*/
#ifndef _LINUX_TRACE_IRQFLAGS_H
#define _LINUX_TRACE_IRQFLAGS_H
#ifdef CONFIG_TRACE_IRQFLAGS
extern void trace_hardirqs_on(void);
extern void trace_hardirqs_off(void);
extern void trace_softirqs_on(unsigned long ip);
extern void trace_softirqs_off(unsigned long ip);
# define trace_hardirq_context(p) ((p)->hardirq_context)
# define trace_softirq_context(p) ((p)->softirq_context)
# define trace_hardirqs_enabled(p) ((p)->hardirqs_enabled)
# define trace_softirqs_enabled(p) ((p)->softirqs_enabled)
# define trace_hardirq_enter() do { current->hardirq_context++; } while (0)
# define trace_hardirq_exit() do { current->hardirq_context--; } while (0)
# define trace_softirq_enter() do { current->softirq_context++; } while (0)
# define trace_softirq_exit() do { current->softirq_context--; } while (0)
# define INIT_TRACE_IRQFLAGS .softirqs_enabled = 1,
#else
# define trace_hardirqs_on() do { } while (0)
# define trace_hardirqs_off() do { } while (0)
# define trace_softirqs_on(ip) do { } while (0)
# define trace_softirqs_off(ip) do { } while (0)
# define trace_hardirq_context(p) 0
# define trace_softirq_context(p) 0
# define trace_hardirqs_enabled(p) 0
# define trace_softirqs_enabled(p) 0
# define trace_hardirq_enter() do { } while (0)
# define trace_hardirq_exit() do { } while (0)
# define trace_softirq_enter() do { } while (0)
# define trace_softirq_exit() do { } while (0)
# define INIT_TRACE_IRQFLAGS
#endif
#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
#include <asm/irqflags.h>
#define local_irq_enable() \
do { trace_hardirqs_on(); raw_local_irq_enable(); } while (0)
#define local_irq_disable() \
do { raw_local_irq_disable(); trace_hardirqs_off(); } while (0)
#define local_irq_save(flags) \
do { raw_local_irq_save(flags); trace_hardirqs_off(); } while (0)
#define local_irq_restore(flags) \
do { \
if (raw_irqs_disabled_flags(flags)) { \
raw_local_irq_restore(flags); \
trace_hardirqs_off(); \
} else { \
trace_hardirqs_on(); \
raw_local_irq_restore(flags); \
} \
} while (0)
#else /* !CONFIG_TRACE_IRQFLAGS_SUPPORT */
/*
* The local_irq_*() APIs are equal to the raw_local_irq*()
* if !TRACE_IRQFLAGS.
*/
# define raw_local_irq_disable() local_irq_disable()
# define raw_local_irq_enable() local_irq_enable()
# define raw_local_irq_save(flags) local_irq_save(flags)
# define raw_local_irq_restore(flags) local_irq_restore(flags)
#endif /* CONFIG_TRACE_IRQFLAGS_SUPPORT */
#ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
#define safe_halt() \
do { \
trace_hardirqs_on(); \
raw_safe_halt(); \
} while (0)
#define local_save_flags(flags) raw_local_save_flags(flags)
#define irqs_disabled() \
({ \
unsigned long flags; \
\
raw_local_save_flags(flags); \
raw_irqs_disabled_flags(flags); \
})
#define irqs_disabled_flags(flags) raw_irqs_disabled_flags(flags)
#endif /* CONFIG_X86 */
#endif
...@@ -871,6 +871,21 @@ struct task_struct { ...@@ -871,6 +871,21 @@ struct task_struct {
/* mutex deadlock detection */ /* mutex deadlock detection */
struct mutex_waiter *blocked_on; struct mutex_waiter *blocked_on;
#endif #endif
#ifdef CONFIG_TRACE_IRQFLAGS
unsigned int irq_events;
int hardirqs_enabled;
unsigned long hardirq_enable_ip;
unsigned int hardirq_enable_event;
unsigned long hardirq_disable_ip;
unsigned int hardirq_disable_event;
int softirqs_enabled;
unsigned long softirq_disable_ip;
unsigned int softirq_disable_event;
unsigned long softirq_enable_ip;
unsigned int softirq_enable_event;
int hardirq_context;
int softirq_context;
#endif
/* journalling filesystem info */ /* journalling filesystem info */
void *journal_info; void *journal_info;
......
...@@ -968,6 +968,10 @@ static task_t *copy_process(unsigned long clone_flags, ...@@ -968,6 +968,10 @@ static task_t *copy_process(unsigned long clone_flags,
if (!p) if (!p)
goto fork_out; goto fork_out;
#ifdef CONFIG_TRACE_IRQFLAGS
DEBUG_LOCKS_WARN_ON(!p->hardirqs_enabled);
DEBUG_LOCKS_WARN_ON(!p->softirqs_enabled);
#endif
retval = -EAGAIN; retval = -EAGAIN;
if (atomic_read(&p->user->processes) >= if (atomic_read(&p->user->processes) >=
p->signal->rlim[RLIMIT_NPROC].rlim_cur) { p->signal->rlim[RLIMIT_NPROC].rlim_cur) {
...@@ -1042,6 +1046,21 @@ static task_t *copy_process(unsigned long clone_flags, ...@@ -1042,6 +1046,21 @@ static task_t *copy_process(unsigned long clone_flags,
} }
mpol_fix_fork_child_flag(p); mpol_fix_fork_child_flag(p);
#endif #endif
#ifdef CONFIG_TRACE_IRQFLAGS
p->irq_events = 0;
p->hardirqs_enabled = 0;
p->hardirq_enable_ip = 0;
p->hardirq_enable_event = 0;
p->hardirq_disable_ip = _THIS_IP_;
p->hardirq_disable_event = 0;
p->softirqs_enabled = 1;
p->softirq_enable_ip = _THIS_IP_;
p->softirq_enable_event = 0;
p->softirq_disable_ip = 0;
p->softirq_disable_event = 0;
p->hardirq_context = 0;
p->softirq_context = 0;
#endif
rt_mutex_init_task(p); rt_mutex_init_task(p);
......
...@@ -4462,7 +4462,9 @@ int __sched cond_resched_softirq(void) ...@@ -4462,7 +4462,9 @@ int __sched cond_resched_softirq(void)
BUG_ON(!in_softirq()); BUG_ON(!in_softirq());
if (need_resched() && __resched_legal()) { if (need_resched() && __resched_legal()) {
__local_bh_enable(); raw_local_irq_disable();
_local_bh_enable();
raw_local_irq_enable();
__cond_resched(); __cond_resched();
local_bh_disable(); local_bh_disable();
return 1; return 1;
......
...@@ -61,6 +61,119 @@ static inline void wakeup_softirqd(void) ...@@ -61,6 +61,119 @@ static inline void wakeup_softirqd(void)
wake_up_process(tsk); wake_up_process(tsk);
} }
/*
* This one is for softirq.c-internal use,
* where hardirqs are disabled legitimately:
*/
static void __local_bh_disable(unsigned long ip)
{
unsigned long flags;
WARN_ON_ONCE(in_irq());
raw_local_irq_save(flags);
add_preempt_count(SOFTIRQ_OFFSET);
/*
* Were softirqs turned off above:
*/
if (softirq_count() == SOFTIRQ_OFFSET)
trace_softirqs_off(ip);
raw_local_irq_restore(flags);
}
void local_bh_disable(void)
{
__local_bh_disable((unsigned long)__builtin_return_address(0));
}
EXPORT_SYMBOL(local_bh_disable);
void __local_bh_enable(void)
{
WARN_ON_ONCE(in_irq());
/*
* softirqs should never be enabled by __local_bh_enable(),
* it always nests inside local_bh_enable() sections:
*/
WARN_ON_ONCE(softirq_count() == SOFTIRQ_OFFSET);
sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL_GPL(__local_bh_enable);
/*
* Special-case - softirqs can safely be enabled in
* cond_resched_softirq(), or by __do_softirq(),
* without processing still-pending softirqs:
*/
void _local_bh_enable(void)
{
WARN_ON_ONCE(in_irq());
WARN_ON_ONCE(!irqs_disabled());
if (softirq_count() == SOFTIRQ_OFFSET)
trace_softirqs_on((unsigned long)__builtin_return_address(0));
sub_preempt_count(SOFTIRQ_OFFSET);
}
EXPORT_SYMBOL(_local_bh_enable);
void local_bh_enable(void)
{
unsigned long flags;
WARN_ON_ONCE(in_irq());
WARN_ON_ONCE(irqs_disabled());
local_irq_save(flags);
/*
* Are softirqs going to be turned on now:
*/
if (softirq_count() == SOFTIRQ_OFFSET)
trace_softirqs_on((unsigned long)__builtin_return_address(0));
/*
* Keep preemption disabled until we are done with
* softirq processing:
*/
sub_preempt_count(SOFTIRQ_OFFSET - 1);
if (unlikely(!in_interrupt() && local_softirq_pending()))
do_softirq();
dec_preempt_count();
local_irq_restore(flags);
preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable);
void local_bh_enable_ip(unsigned long ip)
{
unsigned long flags;
WARN_ON_ONCE(in_irq());
local_irq_save(flags);
/*
* Are softirqs going to be turned on now:
*/
if (softirq_count() == SOFTIRQ_OFFSET)
trace_softirqs_on(ip);
/*
* Keep preemption disabled until we are done with
* softirq processing:
*/
sub_preempt_count(SOFTIRQ_OFFSET - 1);
if (unlikely(!in_interrupt() && local_softirq_pending()))
do_softirq();
dec_preempt_count();
local_irq_restore(flags);
preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable_ip);
/* /*
* We restart softirq processing MAX_SOFTIRQ_RESTART times, * We restart softirq processing MAX_SOFTIRQ_RESTART times,
* and we fall back to softirqd after that. * and we fall back to softirqd after that.
...@@ -80,8 +193,9 @@ asmlinkage void __do_softirq(void) ...@@ -80,8 +193,9 @@ asmlinkage void __do_softirq(void)
int cpu; int cpu;
pending = local_softirq_pending(); pending = local_softirq_pending();
__local_bh_disable((unsigned long)__builtin_return_address(0));
trace_softirq_enter();
local_bh_disable();
cpu = smp_processor_id(); cpu = smp_processor_id();
restart: restart:
/* Reset the pending bitmask before enabling irqs */ /* Reset the pending bitmask before enabling irqs */
...@@ -109,7 +223,8 @@ asmlinkage void __do_softirq(void) ...@@ -109,7 +223,8 @@ asmlinkage void __do_softirq(void)
if (pending) if (pending)
wakeup_softirqd(); wakeup_softirqd();
__local_bh_enable(); trace_softirq_exit();
_local_bh_enable();
} }
#ifndef __ARCH_HAS_DO_SOFTIRQ #ifndef __ARCH_HAS_DO_SOFTIRQ
...@@ -136,23 +251,6 @@ EXPORT_SYMBOL(do_softirq); ...@@ -136,23 +251,6 @@ EXPORT_SYMBOL(do_softirq);
#endif #endif
void local_bh_enable(void)
{
WARN_ON(irqs_disabled());
/*
* Keep preemption disabled until we are done with
* softirq processing:
*/
sub_preempt_count(SOFTIRQ_OFFSET - 1);
if (unlikely(!in_interrupt() && local_softirq_pending()))
do_softirq();
dec_preempt_count();
preempt_check_resched();
}
EXPORT_SYMBOL(local_bh_enable);
#ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED #ifdef __ARCH_IRQ_EXIT_IRQS_DISABLED
# define invoke_softirq() __do_softirq() # define invoke_softirq() __do_softirq()
#else #else
...@@ -165,6 +263,7 @@ EXPORT_SYMBOL(local_bh_enable); ...@@ -165,6 +263,7 @@ EXPORT_SYMBOL(local_bh_enable);
void irq_exit(void) void irq_exit(void)
{ {
account_system_vtime(current); account_system_vtime(current);
trace_hardirq_exit();
sub_preempt_count(IRQ_EXIT_OFFSET); sub_preempt_count(IRQ_EXIT_OFFSET);
if (!in_interrupt() && local_softirq_pending()) if (!in_interrupt() && local_softirq_pending())
invoke_softirq(); invoke_softirq();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment