trace_branch.c 8.19 KB
Newer Older
1 2 3 4 5 6 7 8
/*
 * unlikely profiler
 *
 * Copyright (C) 2008 Steven Rostedt <srostedt@redhat.com>
 */
#include <linux/kallsyms.h>
#include <linux/seq_file.h>
#include <linux/spinlock.h>
9
#include <linux/irqflags.h>
10 11 12 13 14 15 16
#include <linux/debugfs.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/ftrace.h>
#include <linux/hash.h>
#include <linux/fs.h>
#include <asm/local.h>
17

18
#include "trace.h"
19
#include "trace_stat.h"
20
#include "trace_output.h"
21

22
#ifdef CONFIG_BRANCH_TRACER
23

24
static struct tracer branch_trace;
25 26
static int branch_tracing_enabled __read_mostly;
static DEFINE_MUTEX(branch_tracing_mutex);
27

28
static struct trace_array *branch_tracer;
29 30

static void
31
probe_likely_condition(struct ftrace_branch_data *f, int val, int expect)
32
{
33
	struct trace_array *tr = branch_tracer;
34
	struct ring_buffer_event *event;
35
	struct trace_branch *entry;
36
	unsigned long flags;
37 38 39 40 41 42 43 44 45 46 47 48 49
	int cpu, pc;
	const char *p;

	/*
	 * I would love to save just the ftrace_likely_data pointer, but
	 * this code can also be used by modules. Ugly things can happen
	 * if the module is unloaded, and then we go and read the
	 * pointer.  This is slower, but much safer.
	 */

	if (unlikely(!tr))
		return;

50
	local_irq_save(flags);
51 52 53 54
	cpu = raw_smp_processor_id();
	if (atomic_inc_return(&tr->data[cpu]->disabled) != 1)
		goto out;

55 56 57
	pc = preempt_count();
	event = trace_buffer_lock_reserve(tr, TRACE_BRANCH,
					  sizeof(*entry), flags, pc);
58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
	if (!event)
		goto out;

	entry	= ring_buffer_event_data(event);

	/* Strip off the path, only save the file */
	p = f->file + strlen(f->file);
	while (p >= f->file && *p != '/')
		p--;
	p++;

	strncpy(entry->func, f->func, TRACE_FUNC_SIZE);
	strncpy(entry->file, p, TRACE_FILE_SIZE);
	entry->func[TRACE_FUNC_SIZE] = 0;
	entry->file[TRACE_FILE_SIZE] = 0;
	entry->line = f->line;
	entry->correct = val == expect;

76
	ring_buffer_unlock_commit(tr->buffer, event);
77 78 79

 out:
	atomic_dec(&tr->data[cpu]->disabled);
80
	local_irq_restore(flags);
81 82 83
}

static inline
84
void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
85
{
86
	if (!branch_tracing_enabled)
87 88 89 90 91
		return;

	probe_likely_condition(f, val, expect);
}

92
int enable_branch_tracing(struct trace_array *tr)
93
{
94 95
	mutex_lock(&branch_tracing_mutex);
	branch_tracer = tr;
96 97 98 99 100
	/*
	 * Must be seen before enabling. The reader is a condition
	 * where we do not need a matching rmb()
	 */
	smp_wmb();
101 102
	branch_tracing_enabled++;
	mutex_unlock(&branch_tracing_mutex);
103

104
	return 0;
105 106
}

107
void disable_branch_tracing(void)
108
{
109
	mutex_lock(&branch_tracing_mutex);
110

111
	if (!branch_tracing_enabled)
112 113
		goto out_unlock;

114
	branch_tracing_enabled--;
115 116

 out_unlock:
117
	mutex_unlock(&branch_tracing_mutex);
118
}
119 120 121 122 123 124 125 126 127 128 129

static void start_branch_trace(struct trace_array *tr)
{
	enable_branch_tracing(tr);
}

static void stop_branch_trace(struct trace_array *tr)
{
	disable_branch_tracing();
}

130
static int branch_trace_init(struct trace_array *tr)
131 132
{
	start_branch_trace(tr);
133
	return 0;
134 135 136 137 138 139 140
}

static void branch_trace_reset(struct trace_array *tr)
{
	stop_branch_trace(tr);
}

141 142
static enum print_line_t trace_branch_print(struct trace_iterator *iter,
					    int flags)
143 144 145
{
	struct trace_branch *field;

146
	trace_assign_type(field, iter->ent);
147

148
	if (trace_seq_printf(&iter->seq, "[%s] %s:%s:%d\n",
149 150 151 152 153 154
			     field->correct ? "  ok  " : " MISS ",
			     field->func,
			     field->file,
			     field->line))
		return TRACE_TYPE_PARTIAL_LINE;

155
	return TRACE_TYPE_HANDLED;
156 157
}

158

159
static struct trace_event trace_branch_event = {
160
	.type		= TRACE_BRANCH,
161 162 163
	.trace		= trace_branch_print,
};

164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
static struct tracer branch_trace __read_mostly =
{
	.name		= "branch",
	.init		= branch_trace_init,
	.reset		= branch_trace_reset,
#ifdef CONFIG_FTRACE_SELFTEST
	.selftest	= trace_selftest_startup_branch,
#endif /* CONFIG_FTRACE_SELFTEST */
};

__init static int init_branch_tracer(void)
{
	int ret;

	ret = register_ftrace_event(&trace_branch_event);
	if (!ret) {
		printk(KERN_WARNING "Warning: could not register "
				    "branch events\n");
		return 1;
	}
	return register_tracer(&branch_trace);
}
device_initcall(init_branch_tracer);

188 189
#else
static inline
190
void trace_likely_condition(struct ftrace_branch_data *f, int val, int expect)
191 192
{
}
193
#endif /* CONFIG_BRANCH_TRACER */
194

195
void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect)
196
{
197 198 199 200 201 202 203 204
	/*
	 * I would love to have a trace point here instead, but the
	 * trace point code is so inundated with unlikely and likely
	 * conditions that the recursive nightmare that exists is too
	 * much to try to get working. At least for now.
	 */
	trace_likely_condition(f, val, expect);

205 206 207 208 209 210 211 212
	/* FIXME: Make this atomic! */
	if (val == expect)
		f->correct++;
	else
		f->incorrect++;
}
EXPORT_SYMBOL(ftrace_likely_update);

213 214
extern unsigned long __start_annotated_branch_profile[];
extern unsigned long __stop_annotated_branch_profile[];
215

216
static int annotated_branch_stat_headers(struct seq_file *m)
217
{
218 219 220 221 222 223 224
	seq_printf(m, " correct incorrect  %% ");
	seq_printf(m, "       Function                "
			      "  File              Line\n"
			      " ------- ---------  - "
			      "       --------                "
			      "  ----              ----\n");
	return 0;
225 226
}

227
static inline long get_incorrect_percent(struct ftrace_branch_data *p)
228
{
229
	long percent;
230

231 232 233 234 235
	if (p->correct) {
		percent = p->incorrect * 100;
		percent /= p->correct + p->incorrect;
	} else
		percent = p->incorrect ? 100 : -1;
236

237
	return percent;
238 239
}

240
static int branch_stat_show(struct seq_file *m, void *v)
241
{
242
	struct ftrace_branch_data *p = v;
243
	const char *f;
244
	long percent;
245 246 247 248 249 250 251

	/* Only print the file, not the path */
	f = p->file + strlen(p->file);
	while (f >= p->file && *f != '/')
		f--;
	f++;

252 253 254
	/*
	 * The miss is overlayed on correct, and hit on incorrect.
	 */
255
	percent = get_incorrect_percent(p);
256

257 258 259 260 261
	seq_printf(m, "%8lu %8lu ",  p->correct, p->incorrect);
	if (percent < 0)
		seq_printf(m, "  X ");
	else
		seq_printf(m, "%3ld ", percent);
262 263 264 265
	seq_printf(m, "%-30.30s %-20.20s %d\n", p->func, f, p->line);
	return 0;
}

266 267 268 269
static void *annotated_branch_stat_start(void)
{
	return __start_annotated_branch_profile;
}
270

271 272
static void *
annotated_branch_stat_next(void *v, int idx)
273
{
274
	struct ftrace_branch_data *p = v;
275

276
	++p;
277

278 279 280 281
	if ((void *)p >= (void *)__stop_annotated_branch_profile)
		return NULL;

	return p;
282 283
}

284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300
static int annotated_branch_stat_cmp(void *p1, void *p2)
{
	struct ftrace_branch_data *a = p1;
	struct ftrace_branch_data *b = p2;

	long percent_a, percent_b;

	percent_a = get_incorrect_percent(a);
	percent_b = get_incorrect_percent(b);

	if (percent_a < percent_b)
		return -1;
	if (percent_a > percent_b)
		return 1;
	else
		return 0;
}
301

302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
static struct tracer_stat annotated_branch_stats = {
	.name = "branch_annotated",
	.stat_start = annotated_branch_stat_start,
	.stat_next = annotated_branch_stat_next,
	.stat_cmp = annotated_branch_stat_cmp,
	.stat_headers = annotated_branch_stat_headers,
	.stat_show = branch_stat_show
};

__init static int init_annotated_branch_stats(void)
{
	int ret;

	ret = register_stat_tracer(&annotated_branch_stats);
	if (!ret) {
		printk(KERN_WARNING "Warning: could not register "
				    "annotated branches stats\n");
		return 1;
	}
	return 0;
}
fs_initcall(init_annotated_branch_stats);

325 326
#ifdef CONFIG_PROFILE_ALL_BRANCHES

327 328
extern unsigned long __start_branch_profile[];
extern unsigned long __stop_branch_profile[];
329

330 331 332 333 334 335 336 337 338 339
static int all_branch_stat_headers(struct seq_file *m)
{
	seq_printf(m, "   miss      hit    %% ");
	seq_printf(m, "       Function                "
			      "  File              Line\n"
			      " ------- ---------  - "
			      "       --------                "
			      "  ----              ----\n");
	return 0;
}
340

341
static void *all_branch_stat_start(void)
342
{
343 344 345 346 347 348 349
	return __start_branch_profile;
}

static void *
all_branch_stat_next(void *v, int idx)
{
	struct ftrace_branch_data *p = v;
350

351
	++p;
352

353 354
	if ((void *)p >= (void *)__stop_branch_profile)
		return NULL;
355

356 357
	return p;
}
358

359 360
static struct tracer_stat all_branch_stats = {
	.name = "branch_all",
361 362 363
	.stat_start = all_branch_stat_start,
	.stat_next = all_branch_stat_next,
	.stat_headers = all_branch_stat_headers,
364
	.stat_show = branch_stat_show
365
};
366

367
__init static int all_annotated_branch_stats(void)
368 369
{
	int ret;
370 371

	ret = register_stat_tracer(&all_branch_stats);
372
	if (!ret) {
373 374
		printk(KERN_WARNING "Warning: could not register "
				    "all branches stats\n");
375 376
		return 1;
	}
377
	return 0;
378
}
379 380
fs_initcall(all_annotated_branch_stats);
#endif /* CONFIG_PROFILE_ALL_BRANCHES */