core.c 158 KB
Newer Older
1
/*
2
 * Performance events core code:
3
 *
4
 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 6
 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7
 *  Copyright    2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8
 *
9
 * For licensing details see kernel-base/COPYING
10 11 12
 */

#include <linux/fs.h>
13
#include <linux/mm.h>
14 15
#include <linux/cpu.h>
#include <linux/smp.h>
Peter Zijlstra's avatar
Peter Zijlstra committed
16
#include <linux/idr.h>
17
#include <linux/file.h>
18
#include <linux/poll.h>
19
#include <linux/slab.h>
20
#include <linux/hash.h>
21
#include <linux/sysfs.h>
22
#include <linux/dcache.h>
23
#include <linux/percpu.h>
24
#include <linux/ptrace.h>
25
#include <linux/reboot.h>
26
#include <linux/vmstat.h>
Peter Zijlstra's avatar
Peter Zijlstra committed
27
#include <linux/device.h>
28
#include <linux/export.h>
29
#include <linux/vmalloc.h>
30 31
#include <linux/hardirq.h>
#include <linux/rculist.h>
32 33 34
#include <linux/uaccess.h>
#include <linux/syscalls.h>
#include <linux/anon_inodes.h>
35
#include <linux/kernel_stat.h>
36
#include <linux/perf_event.h>
37
#include <linux/ftrace_event.h>
38
#include <linux/hw_breakpoint.h>
39

40 41
#include "internal.h"

42 43
#include <asm/irq_regs.h>

44
struct remote_function_call {
45 46 47 48
	struct task_struct	*p;
	int			(*func)(void *info);
	void			*info;
	int			ret;
49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81
};

static void remote_function(void *data)
{
	struct remote_function_call *tfc = data;
	struct task_struct *p = tfc->p;

	if (p) {
		tfc->ret = -EAGAIN;
		if (task_cpu(p) != smp_processor_id() || !task_curr(p))
			return;
	}

	tfc->ret = tfc->func(tfc->info);
}

/**
 * task_function_call - call a function on the cpu on which a task runs
 * @p:		the task to evaluate
 * @func:	the function to be called
 * @info:	the function call argument
 *
 * Calls the function @func when the task is currently running. This might
 * be on the current CPU, which just calls the function directly
 *
 * returns: @func return value, or
 *	    -ESRCH  - when the process isn't running
 *	    -EAGAIN - when the process moved away
 */
static int
task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
{
	struct remote_function_call data = {
82 83 84 85
		.p	= p,
		.func	= func,
		.info	= info,
		.ret	= -ESRCH, /* No such (running) process */
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
	};

	if (task_curr(p))
		smp_call_function_single(task_cpu(p), remote_function, &data, 1);

	return data.ret;
}

/**
 * cpu_function_call - call a function on the cpu
 * @func:	the function to be called
 * @info:	the function call argument
 *
 * Calls the function @func on the remote cpu.
 *
 * returns: @func return value or -ENXIO when the cpu is offline
 */
static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
{
	struct remote_function_call data = {
106 107 108 109
		.p	= NULL,
		.func	= func,
		.info	= info,
		.ret	= -ENXIO, /* No such CPU */
110 111 112 113 114 115 116
	};

	smp_call_function_single(cpu, remote_function, &data, 1);

	return data.ret;
}

Stephane Eranian's avatar
Stephane Eranian committed
117 118 119 120
#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
		       PERF_FLAG_FD_OUTPUT  |\
		       PERF_FLAG_PID_CGROUP)

121 122 123 124 125 126
enum event_type_t {
	EVENT_FLEXIBLE = 0x1,
	EVENT_PINNED = 0x2,
	EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
};

Stephane Eranian's avatar
Stephane Eranian committed
127 128 129 130
/*
 * perf_sched_events : >0 events exist
 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
 */
131
struct jump_label_key perf_sched_events __read_mostly;
Stephane Eranian's avatar
Stephane Eranian committed
132 133
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);

134 135 136
static atomic_t nr_mmap_events __read_mostly;
static atomic_t nr_comm_events __read_mostly;
static atomic_t nr_task_events __read_mostly;
137

138 139 140 141
static LIST_HEAD(pmus);
static DEFINE_MUTEX(pmus_lock);
static struct srcu_struct pmus_srcu;

142
/*
143
 * perf event paranoia level:
144 145
 *  -1 - not paranoid at all
 *   0 - disallow raw tracepoint access for unpriv
146
 *   1 - disallow cpu events for unpriv
147
 *   2 - disallow kernel profiling for unpriv
148
 */
149
int sysctl_perf_event_paranoid __read_mostly = 1;
150

151 152
/* Minimum for 512 kiB + 1 user control page */
int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
153 154

/*
155
 * max perf event sample rate
156
 */
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174
#define DEFAULT_MAX_SAMPLE_RATE 100000
int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
static int max_samples_per_tick __read_mostly =
	DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);

int perf_proc_update_handler(struct ctl_table *table, int write,
		void __user *buffer, size_t *lenp,
		loff_t *ppos)
{
	int ret = proc_dointvec(table, write, buffer, lenp, ppos);

	if (ret || !write)
		return ret;

	max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);

	return 0;
}
175

176
static atomic64_t perf_event_id;
177

178 179 180 181
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type);

static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
Stephane Eranian's avatar
Stephane Eranian committed
182 183 184 185 186
			     enum event_type_t event_type,
			     struct task_struct *task);

static void update_context_time(struct perf_event_context *ctx);
static u64 perf_event_time(struct perf_event *event);
187

188
void __weak perf_event_print_debug(void)	{ }
189

190
extern __weak const char *perf_pmu_name(void)
191
{
192
	return "pmu";
193 194
}

195 196 197 198 199
static inline u64 perf_clock(void)
{
	return local_clock();
}

Stephane Eranian's avatar
Stephane Eranian committed
200 201 202 203 204 205
static inline struct perf_cpu_context *
__get_cpu_context(struct perf_event_context *ctx)
{
	return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
}

206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221
static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
			  struct perf_event_context *ctx)
{
	raw_spin_lock(&cpuctx->ctx.lock);
	if (ctx)
		raw_spin_lock(&ctx->lock);
}

static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
			    struct perf_event_context *ctx)
{
	if (ctx)
		raw_spin_unlock(&ctx->lock);
	raw_spin_unlock(&cpuctx->ctx.lock);
}

Stephane Eranian's avatar
Stephane Eranian committed
222 223
#ifdef CONFIG_CGROUP_PERF

224 225 226 227 228
/*
 * Must ensure cgroup is pinned (css_get) before calling
 * this function. In other words, we cannot call this function
 * if there is no cgroup event for the current CPU context.
 */
Stephane Eranian's avatar
Stephane Eranian committed
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295
static inline struct perf_cgroup *
perf_cgroup_from_task(struct task_struct *task)
{
	return container_of(task_subsys_state(task, perf_subsys_id),
			struct perf_cgroup, css);
}

static inline bool
perf_cgroup_match(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);

	return !event->cgrp || event->cgrp == cpuctx->cgrp;
}

static inline void perf_get_cgroup(struct perf_event *event)
{
	css_get(&event->cgrp->css);
}

static inline void perf_put_cgroup(struct perf_event *event)
{
	css_put(&event->cgrp->css);
}

static inline void perf_detach_cgroup(struct perf_event *event)
{
	perf_put_cgroup(event);
	event->cgrp = NULL;
}

static inline int is_cgroup_event(struct perf_event *event)
{
	return event->cgrp != NULL;
}

static inline u64 perf_cgroup_event_time(struct perf_event *event)
{
	struct perf_cgroup_info *t;

	t = per_cpu_ptr(event->cgrp->info, event->cpu);
	return t->time;
}

static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
{
	struct perf_cgroup_info *info;
	u64 now;

	now = perf_clock();

	info = this_cpu_ptr(cgrp->info);

	info->time += now - info->timestamp;
	info->timestamp = now;
}

static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
{
	struct perf_cgroup *cgrp_out = cpuctx->cgrp;
	if (cgrp_out)
		__update_cgrp_time(cgrp_out);
}

static inline void update_cgrp_time_from_event(struct perf_event *event)
{
296 297
	struct perf_cgroup *cgrp;

Stephane Eranian's avatar
Stephane Eranian committed
298
	/*
299 300
	 * ensure we access cgroup data only when needed and
	 * when we know the cgroup is pinned (css_get)
Stephane Eranian's avatar
Stephane Eranian committed
301
	 */
302
	if (!is_cgroup_event(event))
Stephane Eranian's avatar
Stephane Eranian committed
303 304
		return;

305 306 307 308 309 310
	cgrp = perf_cgroup_from_task(current);
	/*
	 * Do not update time when cgroup is not active
	 */
	if (cgrp == event->cgrp)
		__update_cgrp_time(event->cgrp);
Stephane Eranian's avatar
Stephane Eranian committed
311 312 313
}

static inline void
314 315
perf_cgroup_set_timestamp(struct task_struct *task,
			  struct perf_event_context *ctx)
Stephane Eranian's avatar
Stephane Eranian committed
316 317 318 319
{
	struct perf_cgroup *cgrp;
	struct perf_cgroup_info *info;

320 321 322 323 324 325
	/*
	 * ctx->lock held by caller
	 * ensure we do not access cgroup data
	 * unless we have the cgroup pinned (css_get)
	 */
	if (!task || !ctx->nr_cgroups)
Stephane Eranian's avatar
Stephane Eranian committed
326 327 328 329
		return;

	cgrp = perf_cgroup_from_task(task);
	info = this_cpu_ptr(cgrp->info);
330
	info->timestamp = ctx->timestamp;
Stephane Eranian's avatar
Stephane Eranian committed
331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371
}

#define PERF_CGROUP_SWOUT	0x1 /* cgroup switch out every event */
#define PERF_CGROUP_SWIN	0x2 /* cgroup switch in events based on task */

/*
 * reschedule events based on the cgroup constraint of task.
 *
 * mode SWOUT : schedule out everything
 * mode SWIN : schedule in based on cgroup for next
 */
void perf_cgroup_switch(struct task_struct *task, int mode)
{
	struct perf_cpu_context *cpuctx;
	struct pmu *pmu;
	unsigned long flags;

	/*
	 * disable interrupts to avoid geting nr_cgroup
	 * changes via __perf_event_disable(). Also
	 * avoids preemption.
	 */
	local_irq_save(flags);

	/*
	 * we reschedule only in the presence of cgroup
	 * constrained events.
	 */
	rcu_read_lock();

	list_for_each_entry_rcu(pmu, &pmus, entry) {
		cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);

		/*
		 * perf_cgroup_events says at least one
		 * context on this CPU has cgroup events.
		 *
		 * ctx->nr_cgroups reports the number of cgroup
		 * events for a context.
		 */
		if (cpuctx->ctx.nr_cgroups > 0) {
372 373
			perf_ctx_lock(cpuctx, cpuctx->task_ctx);
			perf_pmu_disable(cpuctx->ctx.pmu);
Stephane Eranian's avatar
Stephane Eranian committed
374 375 376 377 378 379 380 381 382 383 384

			if (mode & PERF_CGROUP_SWOUT) {
				cpu_ctx_sched_out(cpuctx, EVENT_ALL);
				/*
				 * must not be done before ctxswout due
				 * to event_filter_match() in event_sched_out()
				 */
				cpuctx->cgrp = NULL;
			}

			if (mode & PERF_CGROUP_SWIN) {
385
				WARN_ON_ONCE(cpuctx->cgrp);
Stephane Eranian's avatar
Stephane Eranian committed
386 387 388 389 390 391 392
				/* set cgrp before ctxsw in to
				 * allow event_filter_match() to not
				 * have to pass task around
				 */
				cpuctx->cgrp = perf_cgroup_from_task(task);
				cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
			}
393 394
			perf_pmu_enable(cpuctx->ctx.pmu);
			perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
Stephane Eranian's avatar
Stephane Eranian committed
395 396 397 398 399 400 401 402
		}
	}

	rcu_read_unlock();

	local_irq_restore(flags);
}

403 404
static inline void perf_cgroup_sched_out(struct task_struct *task,
					 struct task_struct *next)
Stephane Eranian's avatar
Stephane Eranian committed
405
{
406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427
	struct perf_cgroup *cgrp1;
	struct perf_cgroup *cgrp2 = NULL;

	/*
	 * we come here when we know perf_cgroup_events > 0
	 */
	cgrp1 = perf_cgroup_from_task(task);

	/*
	 * next is NULL when called from perf_event_enable_on_exec()
	 * that will systematically cause a cgroup_switch()
	 */
	if (next)
		cgrp2 = perf_cgroup_from_task(next);

	/*
	 * only schedule out current cgroup events if we know
	 * that we are switching to a different cgroup. Otherwise,
	 * do no touch the cgroup events.
	 */
	if (cgrp1 != cgrp2)
		perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
Stephane Eranian's avatar
Stephane Eranian committed
428 429
}

430 431
static inline void perf_cgroup_sched_in(struct task_struct *prev,
					struct task_struct *task)
Stephane Eranian's avatar
Stephane Eranian committed
432
{
433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450
	struct perf_cgroup *cgrp1;
	struct perf_cgroup *cgrp2 = NULL;

	/*
	 * we come here when we know perf_cgroup_events > 0
	 */
	cgrp1 = perf_cgroup_from_task(task);

	/* prev can never be NULL */
	cgrp2 = perf_cgroup_from_task(prev);

	/*
	 * only need to schedule in cgroup events if we are changing
	 * cgroup during ctxsw. Cgroup events were not scheduled
	 * out of ctxsw out if that was not the case.
	 */
	if (cgrp1 != cgrp2)
		perf_cgroup_switch(task, PERF_CGROUP_SWIN);
Stephane Eranian's avatar
Stephane Eranian committed
451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466
}

static inline int perf_cgroup_connect(int fd, struct perf_event *event,
				      struct perf_event_attr *attr,
				      struct perf_event *group_leader)
{
	struct perf_cgroup *cgrp;
	struct cgroup_subsys_state *css;
	struct file *file;
	int ret = 0, fput_needed;

	file = fget_light(fd, &fput_needed);
	if (!file)
		return -EBADF;

	css = cgroup_css_from_dir(file, perf_subsys_id);
467 468 469 470
	if (IS_ERR(css)) {
		ret = PTR_ERR(css);
		goto out;
	}
Stephane Eranian's avatar
Stephane Eranian committed
471 472 473 474

	cgrp = container_of(css, struct perf_cgroup, css);
	event->cgrp = cgrp;

475 476 477
	/* must be done before we fput() the file */
	perf_get_cgroup(event);

Stephane Eranian's avatar
Stephane Eranian committed
478 479 480 481 482 483 484 485 486
	/*
	 * all events in a group must monitor
	 * the same cgroup because a task belongs
	 * to only one perf cgroup at a time
	 */
	if (group_leader && group_leader->cgrp != cgrp) {
		perf_detach_cgroup(event);
		ret = -EINVAL;
	}
487
out:
Stephane Eranian's avatar
Stephane Eranian committed
488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561
	fput_light(file, fput_needed);
	return ret;
}

static inline void
perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
{
	struct perf_cgroup_info *t;
	t = per_cpu_ptr(event->cgrp->info, event->cpu);
	event->shadow_ctx_time = now - t->timestamp;
}

static inline void
perf_cgroup_defer_enabled(struct perf_event *event)
{
	/*
	 * when the current task's perf cgroup does not match
	 * the event's, we need to remember to call the
	 * perf_mark_enable() function the first time a task with
	 * a matching perf cgroup is scheduled in.
	 */
	if (is_cgroup_event(event) && !perf_cgroup_match(event))
		event->cgrp_defer_enabled = 1;
}

static inline void
perf_cgroup_mark_enabled(struct perf_event *event,
			 struct perf_event_context *ctx)
{
	struct perf_event *sub;
	u64 tstamp = perf_event_time(event);

	if (!event->cgrp_defer_enabled)
		return;

	event->cgrp_defer_enabled = 0;

	event->tstamp_enabled = tstamp - event->total_time_enabled;
	list_for_each_entry(sub, &event->sibling_list, group_entry) {
		if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
			sub->cgrp_defer_enabled = 0;
		}
	}
}
#else /* !CONFIG_CGROUP_PERF */

static inline bool
perf_cgroup_match(struct perf_event *event)
{
	return true;
}

static inline void perf_detach_cgroup(struct perf_event *event)
{}

static inline int is_cgroup_event(struct perf_event *event)
{
	return 0;
}

static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
{
	return 0;
}

static inline void update_cgrp_time_from_event(struct perf_event *event)
{
}

static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
{
}

562 563
static inline void perf_cgroup_sched_out(struct task_struct *task,
					 struct task_struct *next)
Stephane Eranian's avatar
Stephane Eranian committed
564 565 566
{
}

567 568
static inline void perf_cgroup_sched_in(struct task_struct *prev,
					struct task_struct *task)
Stephane Eranian's avatar
Stephane Eranian committed
569 570 571 572 573 574 575 576 577 578 579
{
}

static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
				      struct perf_event_attr *attr,
				      struct perf_event *group_leader)
{
	return -EINVAL;
}

static inline void
580 581
perf_cgroup_set_timestamp(struct task_struct *task,
			  struct perf_event_context *ctx)
Stephane Eranian's avatar
Stephane Eranian committed
582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611
{
}

void
perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
{
}

static inline void
perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
{
}

static inline u64 perf_cgroup_event_time(struct perf_event *event)
{
	return 0;
}

static inline void
perf_cgroup_defer_enabled(struct perf_event *event)
{
}

static inline void
perf_cgroup_mark_enabled(struct perf_event *event,
			 struct perf_event_context *ctx)
{
}
#endif

Peter Zijlstra's avatar
Peter Zijlstra committed
612
void perf_pmu_disable(struct pmu *pmu)
613
{
Peter Zijlstra's avatar
Peter Zijlstra committed
614 615 616
	int *count = this_cpu_ptr(pmu->pmu_disable_count);
	if (!(*count)++)
		pmu->pmu_disable(pmu);
617 618
}

Peter Zijlstra's avatar
Peter Zijlstra committed
619
void perf_pmu_enable(struct pmu *pmu)
620
{
Peter Zijlstra's avatar
Peter Zijlstra committed
621 622 623
	int *count = this_cpu_ptr(pmu->pmu_disable_count);
	if (!--(*count))
		pmu->pmu_enable(pmu);
624 625
}

626 627 628 629 630 631 632
static DEFINE_PER_CPU(struct list_head, rotation_list);

/*
 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
 * because they're strictly cpu affine and rotate_start is called with IRQs
 * disabled, while rotate_context is called from IRQ context.
 */
633
static void perf_pmu_rotate_start(struct pmu *pmu)
634
{
635
	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
636
	struct list_head *head = &__get_cpu_var(rotation_list);
637

638
	WARN_ON(!irqs_disabled());
639

640 641
	if (list_empty(&cpuctx->rotation_list))
		list_add(&cpuctx->rotation_list, head);
642 643
}

644
static void get_ctx(struct perf_event_context *ctx)
645
{
646
	WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
647 648
}

649
static void put_ctx(struct perf_event_context *ctx)
650
{
651 652 653
	if (atomic_dec_and_test(&ctx->refcount)) {
		if (ctx->parent_ctx)
			put_ctx(ctx->parent_ctx);
654 655
		if (ctx->task)
			put_task_struct(ctx->task);
656
		kfree_rcu(ctx, rcu_head);
657
	}
658 659
}

660
static void unclone_ctx(struct perf_event_context *ctx)
661 662 663 664 665 666 667
{
	if (ctx->parent_ctx) {
		put_ctx(ctx->parent_ctx);
		ctx->parent_ctx = NULL;
	}
}

668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689
static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
{
	/*
	 * only top level events have the pid namespace they were created in
	 */
	if (event->parent)
		event = event->parent;

	return task_tgid_nr_ns(p, event->ns);
}

static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
{
	/*
	 * only top level events have the pid namespace they were created in
	 */
	if (event->parent)
		event = event->parent;

	return task_pid_nr_ns(p, event->ns);
}

690
/*
691
 * If we inherit events we want to return the parent event id
692 693
 * to userspace.
 */
694
static u64 primary_event_id(struct perf_event *event)
695
{
696
	u64 id = event->id;
697

698 699
	if (event->parent)
		id = event->parent->id;
700 701 702 703

	return id;
}

704
/*
705
 * Get the perf_event_context for a task and lock it.
706 707 708
 * This has to cope with with the fact that until it is locked,
 * the context could get moved to another task.
 */
709
static struct perf_event_context *
710
perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
711
{
712
	struct perf_event_context *ctx;
713 714

	rcu_read_lock();
Peter Zijlstra's avatar
Peter Zijlstra committed
715
retry:
716
	ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
717 718 719 720
	if (ctx) {
		/*
		 * If this context is a clone of another, it might
		 * get swapped for another underneath us by
721
		 * perf_event_task_sched_out, though the
722 723 724 725 726 727
		 * rcu_read_lock() protects us from any context
		 * getting freed.  Lock the context and check if it
		 * got swapped before we could get the lock, and retry
		 * if so.  If we locked the right context, then it
		 * can't get swapped on us any more.
		 */
728
		raw_spin_lock_irqsave(&ctx->lock, *flags);
729
		if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
730
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
731 732
			goto retry;
		}
733 734

		if (!atomic_inc_not_zero(&ctx->refcount)) {
735
			raw_spin_unlock_irqrestore(&ctx->lock, *flags);
736 737
			ctx = NULL;
		}
738 739 740 741 742 743 744 745 746 747
	}
	rcu_read_unlock();
	return ctx;
}

/*
 * Get the context for a task and increment its pin_count so it
 * can't get swapped to another task.  This also increments its
 * reference count so that the context can't get freed.
 */
748 749
static struct perf_event_context *
perf_pin_task_context(struct task_struct *task, int ctxn)
750
{
751
	struct perf_event_context *ctx;
752 753
	unsigned long flags;

754
	ctx = perf_lock_task_context(task, ctxn, &flags);
755 756
	if (ctx) {
		++ctx->pin_count;
757
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
758 759 760 761
	}
	return ctx;
}

762
static void perf_unpin_context(struct perf_event_context *ctx)
763 764 765
{
	unsigned long flags;

766
	raw_spin_lock_irqsave(&ctx->lock, flags);
767
	--ctx->pin_count;
768
	raw_spin_unlock_irqrestore(&ctx->lock, flags);
769 770
}

771 772 773 774 775 776 777 778 779 780 781
/*
 * Update the record of the current time in a context.
 */
static void update_context_time(struct perf_event_context *ctx)
{
	u64 now = perf_clock();

	ctx->time += now - ctx->timestamp;
	ctx->timestamp = now;
}

782 783 784
static u64 perf_event_time(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
Stephane Eranian's avatar
Stephane Eranian committed
785 786 787 788

	if (is_cgroup_event(event))
		return perf_cgroup_event_time(event);

789 790 791
	return ctx ? ctx->time : 0;
}

792 793
/*
 * Update the total_time_enabled and total_time_running fields for a event.
794
 * The caller of this function needs to hold the ctx->lock.
795 796 797 798 799 800 801 802 803
 */
static void update_event_times(struct perf_event *event)
{
	struct perf_event_context *ctx = event->ctx;
	u64 run_end;

	if (event->state < PERF_EVENT_STATE_INACTIVE ||
	    event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
		return;
Stephane Eranian's avatar
Stephane Eranian committed
804 805 806 807 808 809 810 811 812 813 814
	/*
	 * in cgroup mode, time_enabled represents
	 * the time the event was enabled AND active
	 * tasks were in the monitored cgroup. This is
	 * independent of the activity of the context as
	 * there may be a mix of cgroup and non-cgroup events.
	 *
	 * That is why we treat cgroup events differently
	 * here.
	 */
	if (is_cgroup_event(event))
815
		run_end = perf_event_time(event);
Stephane Eranian's avatar
Stephane Eranian committed
816 817
	else if (ctx->is_active)
		run_end = ctx->time;
818 819 820 821
	else
		run_end = event->tstamp_stopped;

	event->total_time_enabled = run_end - event->tstamp_enabled;
822 823 824 825

	if (event->state == PERF_EVENT_STATE_INACTIVE)
		run_end = event->tstamp_stopped;
	else
826
		run_end = perf_event_time(event);
827 828

	event->total_time_running = run_end - event->tstamp_running;
Stephane Eranian's avatar
Stephane Eranian committed
829

830 831
}

832 833 834 835 836 837 838 839 840 841 842 843
/*
 * Update total_time_enabled and total_time_running for all events in a group.
 */
static void update_group_times(struct perf_event *leader)
{
	struct perf_event *event;

	update_event_times(leader);
	list_for_each_entry(event, &leader->sibling_list, group_entry)
		update_event_times(event);
}

844 845 846 847 848 849 850 851 852
static struct list_head *
ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
{
	if (event->attr.pinned)
		return &ctx->pinned_groups;
	else
		return &ctx->flexible_groups;
}

853
/*
854
 * Add a event from the lists for its context.
855 856
 * Must be called with ctx->mutex and ctx->lock held.
 */
857
static void
858
list_add_event(struct perf_event *event, struct perf_event_context *ctx)
859
{
860 861
	WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
	event->attach_state |= PERF_ATTACH_CONTEXT;
862 863

	/*
864 865 866
	 * If we're a stand alone event or group leader, we go to the context
	 * list, group events are kept attached to the group so that
	 * perf_group_detach can, at all times, locate all siblings.
867
	 */
868
	if (event->group_leader == event) {
869 870
		struct list_head *list;

871 872 873
		if (is_software_event(event))
			event->group_flags |= PERF_GROUP_SOFTWARE;

874 875
		list = ctx_group_list(event, ctx);
		list_add_tail(&event->group_entry, list);
876
	}
877

878
	if (is_cgroup_event(event))
Stephane Eranian's avatar
Stephane Eranian committed
879 880
		ctx->nr_cgroups++;

881
	list_add_rcu(&event->event_entry, &ctx->event_list);
882
	if (!ctx->nr_events)
883
		perf_pmu_rotate_start(ctx->pmu);
884 885
	ctx->nr_events++;
	if (event->attr.inherit_stat)
886
		ctx->nr_stat++;
887 888
}

889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927
/*
 * Called at perf_event creation and when events are attached/detached from a
 * group.
 */
static void perf_event__read_size(struct perf_event *event)
{
	int entry = sizeof(u64); /* value */
	int size = 0;
	int nr = 1;

	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		size += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		size += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_ID)
		entry += sizeof(u64);

	if (event->attr.read_format & PERF_FORMAT_GROUP) {
		nr += event->group_leader->nr_siblings;
		size += sizeof(u64);
	}

	size += entry * nr;
	event->read_size = size;
}

static void perf_event__header_size(struct perf_event *event)
{
	struct perf_sample_data *data;
	u64 sample_type = event->attr.sample_type;
	u16 size = 0;

	perf_event__read_size(event);

	if (sample_type & PERF_SAMPLE_IP)
		size += sizeof(data->ip);

928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945
	if (sample_type & PERF_SAMPLE_ADDR)
		size += sizeof(data->addr);

	if (sample_type & PERF_SAMPLE_PERIOD)
		size += sizeof(data->period);

	if (sample_type & PERF_SAMPLE_READ)
		size += event->read_size;

	event->header_size = size;
}

static void perf_event__id_header_size(struct perf_event *event)
{
	struct perf_sample_data *data;
	u64 sample_type = event->attr.sample_type;
	u16 size = 0;

946 947 948 949 950 951 952 953 954 955 956 957 958 959 960
	if (sample_type & PERF_SAMPLE_TID)
		size += sizeof(data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		size += sizeof(data->time);

	if (sample_type & PERF_SAMPLE_ID)
		size += sizeof(data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		size += sizeof(data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		size += sizeof(data->cpu_entry);

961
	event->id_header_size = size;
962 963
}

964 965
static void perf_group_attach(struct perf_event *event)
{
966
	struct perf_event *group_leader = event->group_leader, *pos;
967

Peter Zijlstra's avatar
Peter Zijlstra committed
968 969 970 971 972 973
	/*
	 * We can have double attach due to group movement in perf_event_open.
	 */
	if (event->attach_state & PERF_ATTACH_GROUP)
		return;

974 975 976 977 978 979 980 981 982 983 984
	event->attach_state |= PERF_ATTACH_GROUP;

	if (group_leader == event)
		return;

	if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
			!is_software_event(event))
		group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;

	list_add_tail(&event->group_entry, &group_leader->sibling_list);
	group_leader->nr_siblings++;
985 986 987 988 989

	perf_event__header_size(group_leader);

	list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
		perf_event__header_size(pos);
990 991
}

992
/*
993
 * Remove a event from the lists for its context.
994
 * Must be called with ctx->mutex and ctx->lock held.
995
 */
996
static void
997
list_del_event(struct perf_event *event, struct perf_event_context *ctx)
998
{
999
	struct perf_cpu_context *cpuctx;
1000 1001 1002 1003
	/*
	 * We can have double detach due to exit/hot-unplug + close.
	 */
	if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1004
		return;
1005 1006 1007

	event->attach_state &= ~PERF_ATTACH_CONTEXT;

1008
	if (is_cgroup_event(event)) {
Stephane Eranian's avatar
Stephane Eranian committed
1009
		ctx->nr_cgroups--;
1010 1011 1012 1013 1014 1015 1016 1017 1018
		cpuctx = __get_cpu_context(ctx);
		/*
		 * if there are no more cgroup events
		 * then cler cgrp to avoid stale pointer
		 * in update_cgrp_time_from_cpuctx()
		 */
		if (!ctx->nr_cgroups)
			cpuctx->cgrp = NULL;
	}
Stephane Eranian's avatar
Stephane Eranian committed
1019

1020 1021
	ctx->nr_events--;
	if (event->attr.inherit_stat)
1022
		ctx->nr_stat--;
1023

1024
	list_del_rcu(&event->event_entry);
1025

1026 1027
	if (event->group_leader == event)
		list_del_init(&event->group_entry);
1028

1029
	update_group_times(event);
1030 1031 1032 1033 1034 1035 1036 1037 1038 1039

	/*
	 * If event was in error state, then keep it
	 * that way, otherwise bogus counts will be
	 * returned on read(). The only way to get out
	 * of error state is by explicit re-enabling
	 * of the event
	 */
	if (event->state > PERF_EVENT_STATE_OFF)
		event->state = PERF_EVENT_STATE_OFF;
1040 1041
}

1042
static void perf_group_detach(struct perf_event *event)
1043 1044
{
	struct perf_event *sibling, *tmp;
1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
	struct list_head *list = NULL;

	/*
	 * We can have double detach due to exit/hot-unplug + close.
	 */
	if (!(event->attach_state & PERF_ATTACH_GROUP))
		return;

	event->attach_state &= ~PERF_ATTACH_GROUP;

	/*
	 * If this is a sibling, remove it from its group.
	 */
	if (event->group_leader != event) {
		list_del_init(&event->group_entry);
		event->group_leader->nr_siblings--;
1061
		goto out;
1062 1063 1064 1065
	}

	if (!list_empty(&event->group_entry))
		list = &event->group_entry;
1066

1067
	/*
1068 1069
	 * If this was a group event with sibling events then
	 * upgrade the siblings to singleton events by adding them
1070
	 * to whatever list we are on.
1071
	 */
1072
	list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1073 1074
		if (list)
			list_move_tail(&sibling->group_entry, list);
1075
		sibling->group_leader = sibling;
1076 1077 1078

		/* Inherit group flags from the previous leader */
		sibling->group_flags = event->group_flags;
1079
	}
1080 1081 1082 1083 1084 1085

out:
	perf_event__header_size(event->group_leader);

	list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
		perf_event__header_size(tmp);
1086 1087
}

1088 1089 1090
static inline int
event_filter_match(struct perf_event *event)
{
Stephane Eranian's avatar
Stephane Eranian committed
1091 1092
	return (event->cpu == -1 || event->cpu == smp_processor_id())
	    && perf_cgroup_match(event);
1093 1094
}

1095 1096
static void
event_sched_out(struct perf_event *event,
1097
		  struct perf_cpu_context *cpuctx,
1098
		  struct perf_event_context *ctx)
1099
{
1100
	u64 tstamp = perf_event_time(event);
1101 1102 1103 1104 1105 1106 1107 1108 1109
	u64 delta;
	/*
	 * An event which could not be activated because of
	 * filter mismatch still needs to have its timings
	 * maintained, otherwise bogus information is return
	 * via read() for time_enabled, time_running:
	 */
	if (event->state == PERF_EVENT_STATE_INACTIVE
	    && !event_filter_match(event)) {
Stephane Eranian's avatar
Stephane Eranian committed
1110
		delta = tstamp - event->tstamp_stopped;
1111
		event->tstamp_running += delta;
1112
		event->tstamp_stopped = tstamp;
1113 1114
	}

1115
	if (event->state != PERF_EVENT_STATE_ACTIVE)
1116
		return;
1117

1118 1119 1120 1121
	event->state = PERF_EVENT_STATE_INACTIVE;
	if (event->pending_disable) {
		event->pending_disable = 0;
		event->state = PERF_EVENT_STATE_OFF;
1122
	}
1123
	event->tstamp_stopped = tstamp;
1124
	event->pmu->del(event, 0);
1125
	event->oncpu = -1;
1126

1127
	if (!is_software_event(event))
1128 1129
		cpuctx->active_oncpu--;
	ctx->nr_active--;
1130
	if (event->attr.exclusive || !cpuctx->active_oncpu)
1131 1132 1133
		cpuctx->exclusive = 0;
}

1134
static void
1135
group_sched_out(struct perf_event *group_event,
1136
		struct perf_cpu_context *cpuctx,
1137
		struct perf_event_context *ctx)
1138
{
1139
	struct perf_event *event;
1140
	int state = group_event->state;
1141

1142
	event_sched_out(group_event, cpuctx, ctx);
1143 1144 1145 1146

	/*
	 * Schedule out siblings (if any):
	 */
1147 1148
	list_for_each_entry(event, &group_event->sibling_list, group_entry)
		event_sched_out(event, cpuctx, ctx);
1149

1150
	if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1151 1152 1153
		cpuctx->exclusive = 0;
}

1154
/*
1155
 * Cross CPU call to remove a performance event
1156
 *
1157
 * We disable the event on the hardware level first. After that we
1158 1159
 * remove it from the context list.
 */
1160
static int __perf_remove_from_context(void *info)
1161
{
1162 1163
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
1164
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1165

1166
	raw_spin_lock(&ctx->lock);
1167 1168
	event_sched_out(event, cpuctx, ctx);
	list_del_event(event, ctx);
1169 1170 1171 1172
	if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
		ctx->is_active = 0;
		cpuctx->task_ctx = NULL;
	}
1173
	raw_spin_unlock(&ctx->lock);
1174 1175

	return 0;
1176 1177 1178 1179
}


/*
1180
 * Remove the event from a task's (or a CPU's) list of events.
1181
 *
1182
 * CPU events are removed with a smp call. For task events we only
1183
 * call when the task is on a CPU.
1184
 *
1185 1186
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
1187 1188
 * remains valid.  This is OK when called from perf_release since
 * that only calls us on the top-level context, which can't be a clone.
1189
 * When called from perf_event_exit_task, it's OK because the
1190
 * context has been detached from its task.
1191
 */
1192
static void perf_remove_from_context(struct perf_event *event)
1193
{
1194
	struct perf_event_context *ctx = event->ctx;
1195 1196
	struct task_struct *task = ctx->task;

1197 1198
	lockdep_assert_held(&ctx->mutex);

1199 1200
	if (!task) {
		/*
1201
		 * Per cpu events are removed via an smp call and
1202
		 * the removal is always successful.
1203
		 */
1204
		cpu_function_call(event->cpu, __perf_remove_from_context, event);
1205 1206 1207 1208
		return;
	}

retry:
1209 1210
	if (!task_function_call(task, __perf_remove_from_context, event))
		return;
1211

1212
	raw_spin_lock_irq(&ctx->lock);
1213
	/*
1214 1215
	 * If we failed to find a running task, but find the context active now
	 * that we've acquired the ctx->lock, retry.
1216
	 */
1217
	if (ctx->is_active) {
1218
		raw_spin_unlock_irq(&ctx->lock);
1219 1220 1221 1222
		goto retry;
	}

	/*
1223 1224
	 * Since the task isn't running, its safe to remove the event, us
	 * holding the ctx->lock ensures the task won't get scheduled in.
1225
	 */
1226
	list_del_event(event, ctx);
1227
	raw_spin_unlock_irq(&ctx->lock);
1228 1229
}

1230
/*
1231
 * Cross CPU call to disable a performance event
1232
 */
1233
static int __perf_event_disable(void *info)
1234
{
1235 1236
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
1237
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1238 1239

	/*
1240 1241
	 * If this is a per-task event, need to check whether this
	 * event's task is the current task on this cpu.
1242 1243 1244
	 *
	 * Can trigger due to concurrent perf_event_context_sched_out()
	 * flipping contexts around.
1245
	 */
1246
	if (ctx->task && cpuctx->task_ctx != ctx)
1247
		return -EINVAL;
1248

1249
	raw_spin_lock(&ctx->lock);
1250 1251

	/*
1252
	 * If the event is on, turn it off.
1253 1254
	 * If it is in error state, leave it in error state.
	 */
1255
	if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1256
		update_context_time(ctx);
Stephane Eranian's avatar
Stephane Eranian committed
1257
		update_cgrp_time_from_event(event);
1258 1259 1260
		update_group_times(event);
		if (event == event->group_leader)
			group_sched_out(event, cpuctx, ctx);
1261
		else
1262 1263
			event_sched_out(event, cpuctx, ctx);
		event->state = PERF_EVENT_STATE_OFF;
1264 1265
	}

1266
	raw_spin_unlock(&ctx->lock);
1267 1268

	return 0;
1269 1270 1271
}

/*
1272
 * Disable a event.
1273
 *
1274 1275
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
1276
 * remains valid.  This condition is satisifed when called through
1277 1278 1279 1280
 * perf_event_for_each_child or perf_event_for_each because they
 * hold the top-level event's child_mutex, so any descendant that
 * goes to exit will block in sync_child_event.
 * When called from perf_pending_event it's OK because event->ctx
1281
 * is the current context on this CPU and preemption is disabled,
1282
 * hence we can't get into perf_event_task_sched_out for this context.
1283
 */
1284
void perf_event_disable(struct perf_event *event)
1285
{
1286
	struct perf_event_context *ctx = event->ctx;
1287 1288 1289 1290
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
1291
		 * Disable the event on the cpu that it's on
1292
		 */
1293
		cpu_function_call(event->cpu, __perf_event_disable, event);
1294 1295 1296
		return;
	}

Peter Zijlstra's avatar
Peter Zijlstra committed
1297
retry:
1298 1299
	if (!task_function_call(task, __perf_event_disable, event))
		return;
1300

1301
	raw_spin_lock_irq(&ctx->lock);
1302
	/*
1303
	 * If the event is still active, we need to retry the cross-call.
1304
	 */
1305
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
1306
		raw_spin_unlock_irq(&ctx->lock);
1307 1308 1309 1310 1311
		/*
		 * Reload the task pointer, it might have been changed by
		 * a concurrent perf_event_context_sched_out().
		 */
		task = ctx->task;
1312 1313 1314 1315 1316 1317 1318
		goto retry;
	}

	/*
	 * Since we have the lock this context can't be scheduled
	 * in, so we can change the state safely.
	 */
1319 1320 1321
	if (event->state == PERF_EVENT_STATE_INACTIVE) {
		update_group_times(event);
		event->state = PERF_EVENT_STATE_OFF;
1322
	}
1323
	raw_spin_unlock_irq(&ctx->lock);
1324
}
1325
EXPORT_SYMBOL_GPL(perf_event_disable);
1326

Stephane Eranian's avatar
Stephane Eranian committed
1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361
static void perf_set_shadow_time(struct perf_event *event,
				 struct perf_event_context *ctx,
				 u64 tstamp)
{
	/*
	 * use the correct time source for the time snapshot
	 *
	 * We could get by without this by leveraging the
	 * fact that to get to this function, the caller
	 * has most likely already called update_context_time()
	 * and update_cgrp_time_xx() and thus both timestamp
	 * are identical (or very close). Given that tstamp is,
	 * already adjusted for cgroup, we could say that:
	 *    tstamp - ctx->timestamp
	 * is equivalent to
	 *    tstamp - cgrp->timestamp.
	 *
	 * Then, in perf_output_read(), the calculation would
	 * work with no changes because:
	 * - event is guaranteed scheduled in
	 * - no scheduled out in between
	 * - thus the timestamp would be the same
	 *
	 * But this is a bit hairy.
	 *
	 * So instead, we have an explicit cgroup call to remain
	 * within the time time source all along. We believe it
	 * is cleaner and simpler to understand.
	 */
	if (is_cgroup_event(event))
		perf_cgroup_set_shadow_time(event, tstamp);
	else
		event->shadow_ctx_time = tstamp - ctx->timestamp;
}

Peter Zijlstra's avatar
Peter Zijlstra committed
1362 1363 1364 1365
#define MAX_INTERRUPTS (~0ULL)

static void perf_log_throttle(struct perf_event *event, int enable);

1366
static int
1367
event_sched_in(struct perf_event *event,
1368
		 struct perf_cpu_context *cpuctx,
1369
		 struct perf_event_context *ctx)
1370
{
1371 1372
	u64 tstamp = perf_event_time(event);

1373
	if (event->state <= PERF_EVENT_STATE_OFF)
1374 1375
		return 0;

1376
	event->state = PERF_EVENT_STATE_ACTIVE;
1377
	event->oncpu = smp_processor_id();
Peter Zijlstra's avatar
Peter Zijlstra committed
1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388

	/*
	 * Unthrottle events, since we scheduled we might have missed several
	 * ticks already, also for a heavily scheduling task there is little
	 * guarantee it'll get a tick in a timely manner.
	 */
	if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
		perf_log_throttle(event, 1);
		event->hw.interrupts = 0;
	}

1389 1390 1391 1392 1393
	/*
	 * The new state must be visible before we turn it on in the hardware:
	 */
	smp_wmb();

1394
	if (event->pmu->add(event, PERF_EF_START)) {
1395 1396
		event->state = PERF_EVENT_STATE_INACTIVE;
		event->oncpu = -1;
1397 1398 1399
		return -EAGAIN;
	}

1400
	event->tstamp_running += tstamp - event->tstamp_stopped;
1401

Stephane Eranian's avatar
Stephane Eranian committed
1402
	perf_set_shadow_time(event, ctx, tstamp);
1403

1404
	if (!is_software_event(event))
1405
		cpuctx->active_oncpu++;
1406 1407
	ctx->nr_active++;

1408
	if (event->attr.exclusive)
1409 1410
		cpuctx->exclusive = 1;

1411 1412 1413
	return 0;
}

1414
static int
1415
group_sched_in(struct perf_event *group_event,
1416
	       struct perf_cpu_context *cpuctx,
1417
	       struct perf_event_context *ctx)
1418
{
1419
	struct perf_event *event, *partial_group = NULL;
Peter Zijlstra's avatar
Peter Zijlstra committed
1420
	struct pmu *pmu = group_event->pmu;
1421 1422
	u64 now = ctx->time;
	bool simulate = false;
1423

1424
	if (group_event->state == PERF_EVENT_STATE_OFF)
1425 1426
		return 0;

Peter Zijlstra's avatar
Peter Zijlstra committed
1427
	pmu->start_txn(pmu);
1428

1429
	if (event_sched_in(group_event, cpuctx, ctx)) {
Peter Zijlstra's avatar
Peter Zijlstra committed
1430
		pmu->cancel_txn(pmu);
1431
		return -EAGAIN;
1432
	}
1433 1434 1435 1436

	/*
	 * Schedule in siblings as one group (if any):
	 */
1437
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1438
		if (event_sched_in(event, cpuctx, ctx)) {
1439
			partial_group = event;
1440 1441 1442 1443
			goto group_error;
		}
	}

1444
	if (!pmu->commit_txn(pmu))
1445
		return 0;
1446

1447 1448 1449 1450
group_error:
	/*
	 * Groups can be scheduled in as one unit only, so undo any
	 * partial group before returning:
1451 1452 1453 1454 1455 1456 1457 1458 1459 1460
	 * The events up to the failed event are scheduled out normally,
	 * tstamp_stopped will be updated.
	 *
	 * The failed events and the remaining siblings need to have
	 * their timings updated as if they had gone thru event_sched_in()
	 * and event_sched_out(). This is required to get consistent timings
	 * across the group. This also takes care of the case where the group
	 * could never be scheduled by ensuring tstamp_stopped is set to mark
	 * the time the event was actually stopped, such that time delta
	 * calculation in update_event_times() is correct.
1461
	 */
1462 1463
	list_for_each_entry(event, &group_event->sibling_list, group_entry) {
		if (event == partial_group)
1464 1465 1466 1467 1468 1469 1470 1471
			simulate = true;

		if (simulate) {
			event->tstamp_running += now - event->tstamp_stopped;
			event->tstamp_stopped = now;
		} else {
			event_sched_out(event, cpuctx, ctx);
		}
1472
	}
1473
	event_sched_out(group_event, cpuctx, ctx);
1474

Peter Zijlstra's avatar
Peter Zijlstra committed
1475
	pmu->cancel_txn(pmu);
1476

1477 1478 1479
	return -EAGAIN;
}

1480
/*
1481
 * Work out whether we can put this event group on the CPU now.
1482
 */
1483
static int group_can_go_on(struct perf_event *event,
1484 1485 1486 1487
			   struct perf_cpu_context *cpuctx,
			   int can_add_hw)
{
	/*
1488
	 * Groups consisting entirely of software events can always go on.
1489
	 */
1490
	if (event->group_flags & PERF_GROUP_SOFTWARE)
1491 1492 1493
		return 1;
	/*
	 * If an exclusive group is already on, no other hardware
1494
	 * events can go on.
1495 1496 1497 1498 1499
	 */
	if (cpuctx->exclusive)
		return 0;
	/*
	 * If this group is exclusive and there are already
1500
	 * events on the CPU, it can't go on.
1501
	 */
1502
	if (event->attr.exclusive && cpuctx->active_oncpu)
1503 1504 1505 1506 1507 1508 1509 1510
		return 0;
	/*
	 * Otherwise, try to add it if all previous groups were able
	 * to go on.
	 */
	return can_add_hw;
}

1511 1512
static void add_event_to_ctx(struct perf_event *event,
			       struct perf_event_context *ctx)
1513
{
1514 1515
	u64 tstamp = perf_event_time(event);

1516
	list_add_event(event, ctx);
1517
	perf_group_attach(event);
1518 1519 1520
	event->tstamp_enabled = tstamp;
	event->tstamp_running = tstamp;
	event->tstamp_stopped = tstamp;
1521 1522
}

1523 1524 1525 1526 1527 1528
static void task_ctx_sched_out(struct perf_event_context *ctx);
static void
ctx_sched_in(struct perf_event_context *ctx,
	     struct perf_cpu_context *cpuctx,
	     enum event_type_t event_type,
	     struct task_struct *task);
1529

1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541
static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
				struct perf_event_context *ctx,
				struct task_struct *task)
{
	cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
	if (ctx)
		ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
	cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
	if (ctx)
		ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
}

1542
/*
1543
 * Cross CPU call to install and enable a performance event
1544 1545
 *
 * Must be called with ctx->mutex held
1546
 */
1547
static int  __perf_install_in_context(void *info)
1548
{
1549 1550
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
1551
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1552 1553 1554
	struct perf_event_context *task_ctx = cpuctx->task_ctx;
	struct task_struct *task = current;

1555
	perf_ctx_lock(cpuctx, task_ctx);
1556
	perf_pmu_disable(cpuctx->ctx.pmu);
1557 1558

	/*
1559
	 * If there was an active task_ctx schedule it out.
1560
	 */
1561
	if (task_ctx)
1562
		task_ctx_sched_out(task_ctx);
1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576

	/*
	 * If the context we're installing events in is not the
	 * active task_ctx, flip them.
	 */
	if (ctx->task && task_ctx != ctx) {
		if (task_ctx)
			raw_spin_unlock(&task_ctx->lock);
		raw_spin_lock(&ctx->lock);
		task_ctx = ctx;
	}

	if (task_ctx) {
		cpuctx->task_ctx = task_ctx;
1577 1578
		task = task_ctx->task;
	}
1579

1580
	cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1581

1582
	update_context_time(ctx);
Stephane Eranian's avatar
Stephane Eranian committed
1583 1584 1585 1586 1587 1588
	/*
	 * update cgrp time only if current cgrp
	 * matches event->cgrp. Must be done before
	 * calling add_event_to_ctx()
	 */
	update_cgrp_time_from_event(event);
1589

1590
	add_event_to_ctx(event, ctx);
1591

1592
	/*
1593
	 * Schedule everything back in
1594
	 */
1595
	perf_event_sched_in(cpuctx, task_ctx, task);
1596 1597 1598

	perf_pmu_enable(cpuctx->ctx.pmu);
	perf_ctx_unlock(cpuctx, task_ctx);
1599 1600

	return 0;
1601 1602 1603
}

/*
1604
 * Attach a performance event to a context
1605
 *
1606 1607
 * First we add the event to the list with the hardware enable bit
 * in event->hw_config cleared.
1608
 *
1609
 * If the event is attached to a task which is on a CPU we use a smp
1610 1611 1612 1613
 * call to enable it in the task context. The task might have been
 * scheduled away, but we check this in the smp call again.
 */
static void
1614 1615
perf_install_in_context(struct perf_event_context *ctx,
			struct perf_event *event,
1616 1617 1618 1619
			int cpu)
{
	struct task_struct *task = ctx->task;

1620 1621
	lockdep_assert_held(&ctx->mutex);

1622 1623
	event->ctx = ctx;

1624 1625
	if (!task) {
		/*
1626
		 * Per cpu events are installed via an smp call and
1627
		 * the install is always successful.
1628
		 */
1629
		cpu_function_call(cpu, __perf_install_in_context, event);
1630 1631 1632 1633
		return;
	}

retry:
1634 1635
	if (!task_function_call(task, __perf_install_in_context, event))
		return;
1636

1637
	raw_spin_lock_irq(&ctx->lock);
1638
	/*
1639 1640
	 * If we failed to find a running task, but find the context active now
	 * that we've acquired the ctx->lock, retry.
1641
	 */
1642
	if (ctx->is_active) {
1643
		raw_spin_unlock_irq(&ctx->lock);
1644 1645 1646 1647
		goto retry;
	}

	/*
1648 1649
	 * Since the task isn't running, its safe to add the event, us holding
	 * the ctx->lock ensures the task won't get scheduled in.
1650
	 */
1651
	add_event_to_ctx(event, ctx);
1652
	raw_spin_unlock_irq(&ctx->lock);
1653 1654
}

1655
/*
1656
 * Put a event into inactive state and update time fields.
1657 1658 1659 1660 1661 1662
 * Enabling the leader of a group effectively enables all
 * the group members that aren't explicitly disabled, so we
 * have to update their ->tstamp_enabled also.
 * Note: this works for group members as well as group leaders
 * since the non-leader members' sibling_lists will be empty.
 */
1663 1664
static void __perf_event_mark_enabled(struct perf_event *event,
					struct perf_event_context *ctx)
1665
{
1666
	struct perf_event *sub;
1667
	u64 tstamp = perf_event_time(event);
1668

1669
	event->state = PERF_EVENT_STATE_INACTIVE;
1670
	event->tstamp_enabled = tstamp - event->total_time_enabled;
Peter Zijlstra's avatar
Peter Zijlstra committed
1671
	list_for_each_entry(sub, &event->sibling_list, group_entry) {
1672 1673
		if (sub->state >= PERF_EVENT_STATE_INACTIVE)
			sub->tstamp_enabled = tstamp - sub->total_time_enabled;
Peter Zijlstra's avatar
Peter Zijlstra committed
1674
	}
1675 1676
}

1677
/*
1678
 * Cross CPU call to enable a performance event
1679
 */
1680
static int __perf_event_enable(void *info)
1681
{
1682 1683 1684
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *leader = event->group_leader;
1685
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1686
	int err;
1687

1688 1689
	if (WARN_ON_ONCE(!ctx->is_active))
		return -EINVAL;
1690

1691
	raw_spin_lock(&ctx->lock);
1692
	update_context_time(ctx);
1693

1694
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1695
		goto unlock;
Stephane Eranian's avatar
Stephane Eranian committed
1696 1697 1698 1699

	/*
	 * set current task's cgroup time reference point
	 */
1700
	perf_cgroup_set_timestamp(current, ctx);
Stephane Eranian's avatar
Stephane Eranian committed
1701

1702
	__perf_event_mark_enabled(event, ctx);
1703

Stephane Eranian's avatar
Stephane Eranian committed
1704 1705 1706
	if (!event_filter_match(event)) {
		if (is_cgroup_event(event))
			perf_cgroup_defer_enabled(event);
1707
		goto unlock;
Stephane Eranian's avatar
Stephane Eranian committed
1708
	}
1709

1710
	/*
1711
	 * If the event is in a group and isn't the group leader,
1712
	 * then don't put it on unless the group is on.
1713
	 */
1714
	if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
1715
		goto unlock;
1716

1717
	if (!group_can_go_on(event, cpuctx, 1)) {
1718
		err = -EEXIST;
1719
	} else {
1720
		if (event == leader)
1721
			err = group_sched_in(event, cpuctx, ctx);
1722
		else
1723
			err = event_sched_in(event, cpuctx, ctx);
1724
	}
1725 1726 1727

	if (err) {
		/*
1728
		 * If this event can't go on and it's part of a
1729 1730
		 * group, then the whole group has to come off.
		 */
1731
		if (leader != event)
1732
			group_sched_out(leader, cpuctx, ctx);
1733
		if (leader->attr.pinned) {
1734
			update_group_times(leader);
1735
			leader->state = PERF_EVENT_STATE_ERROR;
1736
		}
1737 1738
	}

Peter Zijlstra's avatar
Peter Zijlstra committed
1739
unlock:
1740
	raw_spin_unlock(&ctx->lock);
1741 1742

	return 0;
1743 1744 1745
}

/*
1746
 * Enable a event.
1747
 *
1748 1749
 * If event->ctx is a cloned context, callers must make sure that
 * every task struct that event->ctx->task could possibly point to
1750
 * remains valid.  This condition is satisfied when called through
1751 1752
 * perf_event_for_each_child or perf_event_for_each as described
 * for perf_event_disable.
1753
 */
1754
void perf_event_enable(struct perf_event *event)
1755
{
1756
	struct perf_event_context *ctx = event->ctx;
1757 1758 1759 1760
	struct task_struct *task = ctx->task;

	if (!task) {
		/*
1761
		 * Enable the event on the cpu that it's on
1762
		 */
1763
		cpu_function_call(event->cpu, __perf_event_enable, event);
1764 1765 1766
		return;
	}

1767
	raw_spin_lock_irq(&ctx->lock);
1768
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
1769 1770 1771
		goto out;

	/*
1772 1773
	 * If the event is in error state, clear that first.
	 * That way, if we see the event in error state below, we
1774 1775 1776 1777
	 * know that it has gone back into error state, as distinct
	 * from the task having been scheduled away before the
	 * cross-call arrived.
	 */
1778 1779
	if (event->state == PERF_EVENT_STATE_ERROR)
		event->state = PERF_EVENT_STATE_OFF;
1780

Peter Zijlstra's avatar
Peter Zijlstra committed
1781
retry:
1782 1783 1784 1785 1786
	if (!ctx->is_active) {
		__perf_event_mark_enabled(event, ctx);
		goto out;
	}

1787
	raw_spin_unlock_irq(&ctx->lock);
1788 1789 1790

	if (!task_function_call(task, __perf_event_enable, event))
		return;
1791

1792
	raw_spin_lock_irq(&ctx->lock);
1793 1794

	/*
1795
	 * If the context is active and the event is still off,
1796 1797
	 * we need to retry the cross-call.
	 */
1798 1799 1800 1801 1802 1803
	if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
		/*
		 * task could have been flipped by a concurrent
		 * perf_event_context_sched_out()
		 */
		task = ctx->task;
1804
		goto retry;
1805
	}
1806

Peter Zijlstra's avatar
Peter Zijlstra committed
1807
out:
1808
	raw_spin_unlock_irq(&ctx->lock);
1809
}
1810
EXPORT_SYMBOL_GPL(perf_event_enable);
1811

1812
int perf_event_refresh(struct perf_event *event, int refresh)
1813
{
1814
	/*
1815
	 * not supported on inherited events
1816
	 */
1817
	if (event->attr.inherit || !is_sampling_event(event))
1818 1819
		return -EINVAL;

1820 1821
	atomic_add(refresh, &event->event_limit);
	perf_event_enable(event);
1822 1823

	return 0;
1824
}
1825
EXPORT_SYMBOL_GPL(perf_event_refresh);
1826

1827 1828 1829
static void ctx_sched_out(struct perf_event_context *ctx,
			  struct perf_cpu_context *cpuctx,
			  enum event_type_t event_type)
1830
{
1831
	struct perf_event *event;
1832
	int is_active = ctx->is_active;
1833

1834
	ctx->is_active &= ~event_type;
1835
	if (likely(!ctx->nr_events))
1836 1837
		return;

1838
	update_context_time(ctx);
Stephane Eranian's avatar
Stephane Eranian committed
1839
	update_cgrp_time_from_cpuctx(cpuctx);
1840
	if (!ctx->nr_active)
1841
		return;
1842

1843
	perf_pmu_disable(ctx->pmu);
1844
	if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
1845 1846
		list_for_each_entry(event, &ctx->pinned_groups, group_entry)
			group_sched_out(event, cpuctx, ctx);
Peter Zijlstra's avatar
Peter Zijlstra committed
1847
	}
1848

1849
	if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
1850
		list_for_each_entry(event, &ctx->flexible_groups, group_entry)
1851
			group_sched_out(event, cpuctx, ctx);
Peter Zijlstra's avatar
Peter Zijlstra committed
1852
	}
Peter Zijlstra's avatar
Peter Zijlstra committed
1853
	perf_pmu_enable(ctx->pmu);
1854 1855
}

1856 1857 1858
/*
 * Test whether two contexts are equivalent, i.e. whether they
 * have both been cloned from the same version of the same context
1859 1860 1861 1862
 * and they both have the same number of enabled events.
 * If the number of enabled events is the same, then the set
 * of enabled events should be the same, because these are both
 * inherited contexts, therefore we can't access individual events
1863
 * in them directly with an fd; we can only enable/disable all
1864
 * events via prctl, or enable/disable all events in a family
1865 1866
 * via ioctl, which will have the same effect on both contexts.
 */
1867 1868
static int context_equiv(struct perf_event_context *ctx1,
			 struct perf_event_context *ctx2)
1869 1870
{
	return ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx
1871
		&& ctx1->parent_gen == ctx2->parent_gen
1872
		&& !ctx1->pin_count && !ctx2->pin_count;
1873 1874
}

1875 1876
static void __perf_event_sync_stat(struct perf_event *event,
				     struct perf_event *next_event)
1877 1878 1879
{
	u64 value;

1880
	if (!event->attr.inherit_stat)
1881 1882 1883
		return;

	/*
1884
	 * Update the event value, we cannot use perf_event_read()
1885 1886
	 * because we're in the middle of a context switch and have IRQs
	 * disabled, which upsets smp_call_function_single(), however
1887
	 * we know the event must be on the current CPU, therefore we
1888 1889
	 * don't need to use it.
	 */
1890 1891
	switch (event->state) {
	case PERF_EVENT_STATE_ACTIVE:
1892 1893
		event->pmu->read(event);
		/* fall-through */
1894

1895 1896
	case PERF_EVENT_STATE_INACTIVE:
		update_event_times(event);
1897 1898 1899 1900 1901 1902 1903
		break;

	default:
		break;
	}

	/*
1904
	 * In order to keep per-task stats reliable we need to flip the event
1905 1906
	 * values when we flip the contexts.
	 */
1907 1908 1909
	value = local64_read(&next_event->count);
	value = local64_xchg(&event->count, value);
	local64_set(&next_event->count, value);
1910

1911 1912
	swap(event->total_time_enabled, next_event->total_time_enabled);
	swap(event->total_time_running, next_event->total_time_running);
1913

1914
	/*
1915
	 * Since we swizzled the values, update the user visible data too.
1916
	 */
1917 1918
	perf_event_update_userpage(event);
	perf_event_update_userpage(next_event);
1919 1920 1921 1922 1923
}

#define list_next_entry(pos, member) \
	list_entry(pos->member.next, typeof(*pos), member)

1924 1925
static void perf_event_sync_stat(struct perf_event_context *ctx,
				   struct perf_event_context *next_ctx)
1926
{
1927
	struct perf_event *event, *next_event;
1928 1929 1930 1931

	if (!ctx->nr_stat)
		return;

1932 1933
	update_context_time(ctx);

1934 1935
	event = list_first_entry(&ctx->event_list,
				   struct perf_event, event_entry);
1936

1937 1938
	next_event = list_first_entry(&next_ctx->event_list,
					struct perf_event, event_entry);
1939

1940 1941
	while (&event->event_entry != &ctx->event_list &&
	       &next_event->event_entry != &next_ctx->event_list) {
1942

1943
		__perf_event_sync_stat(event, next_event);
1944

1945 1946
		event = list_next_entry(event, event_entry);
		next_event = list_next_entry(next_event, event_entry);
1947 1948 1949
	}
}

1950 1951
static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
					 struct task_struct *next)
1952
{
1953
	struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
1954 1955
	struct perf_event_context *next_ctx;
	struct perf_event_context *parent;
1956
	struct perf_cpu_context *cpuctx;
1957
	int do_switch = 1;
1958

1959 1960
	if (likely(!ctx))
		return;
1961

1962 1963
	cpuctx = __get_cpu_context(ctx);
	if (!cpuctx->task_ctx)
1964 1965
		return;

1966 1967
	rcu_read_lock();
	parent = rcu_dereference(ctx->parent_ctx);
1968
	next_ctx = next->perf_event_ctxp[ctxn];
1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979
	if (parent && next_ctx &&
	    rcu_dereference(next_ctx->parent_ctx) == parent) {
		/*
		 * Looks like the two contexts are clones, so we might be
		 * able to optimize the context switch.  We lock both
		 * contexts and check that they are clones under the
		 * lock (including re-checking that neither has been
		 * uncloned in the meantime).  It doesn't matter which
		 * order we take the locks because no other cpu could
		 * be trying to lock both of these tasks.
		 */
1980 1981
		raw_spin_lock(&ctx->lock);
		raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
1982
		if (context_equiv(ctx, next_ctx)) {
1983 1984
			/*
			 * XXX do we need a memory barrier of sorts
1985
			 * wrt to rcu_dereference() of perf_event_ctxp
1986
			 */
1987 1988
			task->perf_event_ctxp[ctxn] = next_ctx;
			next->perf_event_ctxp[ctxn] = ctx;
1989 1990 1991
			ctx->task = next;
			next_ctx->task = task;
			do_switch = 0;
1992

1993
			perf_event_sync_stat(ctx, next_ctx);
1994
		}
1995 1996
		raw_spin_unlock(&next_ctx->lock);
		raw_spin_unlock(&ctx->lock);
1997
	}
1998
	rcu_read_unlock();
1999

2000
	if (do_switch) {
2001
		raw_spin_lock(&ctx->lock);
2002
		ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2003
		cpuctx->task_ctx = NULL;
2004
		raw_spin_unlock(&ctx->lock);
2005
	}
2006 2007
}

2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
#define for_each_task_context_nr(ctxn)					\
	for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)

/*
 * Called from scheduler to remove the events of the current task,
 * with interrupts disabled.
 *
 * We stop each event and update the event value in event->count.
 *
 * This does not protect us against NMI, but disable()
 * sets the disabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * not restart the event.
 */
2022 2023
void __perf_event_task_sched_out(struct task_struct *task,
				 struct task_struct *next)
2024 2025 2026 2027 2028
{
	int ctxn;

	for_each_task_context_nr(ctxn)
		perf_event_context_sched_out(task, ctxn, next);
Stephane Eranian's avatar
Stephane Eranian committed
2029 2030 2031 2032 2033 2034 2035

	/*
	 * if cgroup events exist on this CPU, then we need
	 * to check if we have to switch out PMU state.
	 * cgroup event are system-wide mode only
	 */
	if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2036
		perf_cgroup_sched_out(task, next);
2037 2038
}

2039
static void task_ctx_sched_out(struct perf_event_context *ctx)
2040
{
2041
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2042

2043 2044
	if (!cpuctx->task_ctx)
		return;
2045 2046 2047 2048

	if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
		return;

2049
	ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2050 2051 2052
	cpuctx->task_ctx = NULL;
}

2053 2054 2055 2056 2057 2058 2059
/*
 * Called with IRQs disabled
 */
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
			      enum event_type_t event_type)
{
	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2060 2061
}

2062
static void
2063
ctx_pinned_sched_in(struct perf_event_context *ctx,
2064
		    struct perf_cpu_context *cpuctx)
2065
{
2066
	struct perf_event *event;
2067

2068 2069
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		if (event->state <= PERF_EVENT_STATE_OFF)
2070
			continue;
2071
		if (!event_filter_match(event))
2072 2073
			continue;

Stephane Eranian's avatar
Stephane Eranian committed
2074 2075 2076 2077
		/* may need to reset tstamp_enabled */
		if (is_cgroup_event(event))
			perf_cgroup_mark_enabled(event, ctx);

2078
		if (group_can_go_on(event, cpuctx, 1))
2079
			group_sched_in(event, cpuctx, ctx);
2080 2081 2082 2083 2084

		/*
		 * If this pinned group hasn't been scheduled,
		 * put it in error state.
		 */
2085 2086 2087
		if (event->state == PERF_EVENT_STATE_INACTIVE) {
			update_group_times(event);
			event->state = PERF_EVENT_STATE_ERROR;
2088
		}
2089
	}
2090 2091 2092 2093
}

static void
ctx_flexible_sched_in(struct perf_event_context *ctx,
2094
		      struct perf_cpu_context *cpuctx)
2095 2096 2097
{
	struct perf_event *event;
	int can_add_hw = 1;
2098

2099 2100 2101
	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		/* Ignore events in OFF or ERROR state */
		if (event->state <= PERF_EVENT_STATE_OFF)
2102
			continue;
2103 2104
		/*
		 * Listen to the 'cpu' scheduling filter constraint
2105
		 * of events:
2106
		 */
2107
		if (!event_filter_match(event))
2108 2109
			continue;

Stephane Eranian's avatar
Stephane Eranian committed
2110 2111 2112 2113
		/* may need to reset tstamp_enabled */
		if (is_cgroup_event(event))
			perf_cgroup_mark_enabled(event, ctx);

Peter Zijlstra's avatar
Peter Zijlstra committed
2114
		if (group_can_go_on(event, cpuctx, can_add_hw)) {
2115
			if (group_sched_in(event, cpuctx, ctx))
2116
				can_add_hw = 0;
Peter Zijlstra's avatar
Peter Zijlstra committed
2117
		}
2118
	}
2119 2120 2121 2122 2123
}

static void
ctx_sched_in(struct perf_event_context *ctx,
	     struct perf_cpu_context *cpuctx,
Stephane Eranian's avatar
Stephane Eranian committed
2124 2125
	     enum event_type_t event_type,
	     struct task_struct *task)
2126
{
Stephane Eranian's avatar
Stephane Eranian committed
2127
	u64 now;
2128
	int is_active = ctx->is_active;
Stephane Eranian's avatar
Stephane Eranian committed
2129

2130
	ctx->is_active |= event_type;
2131
	if (likely(!ctx->nr_events))
2132
		return;
2133

Stephane Eranian's avatar
Stephane Eranian committed
2134 2135
	now = perf_clock();
	ctx->timestamp = now;
2136
	perf_cgroup_set_timestamp(task, ctx);
2137 2138 2139 2140
	/*
	 * First go through the list and put on any pinned groups
	 * in order to give them the best chance of going on.
	 */
2141
	if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2142
		ctx_pinned_sched_in(ctx, cpuctx);
2143 2144

	/* Then walk through the lower prio flexible groups */
2145
	if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2146
		ctx_flexible_sched_in(ctx, cpuctx);
2147 2148
}

2149
static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
Stephane Eranian's avatar
Stephane Eranian committed
2150 2151
			     enum event_type_t event_type,
			     struct task_struct *task)
2152 2153 2154
{
	struct perf_event_context *ctx = &cpuctx->ctx;

Stephane Eranian's avatar
Stephane Eranian committed
2155
	ctx_sched_in(ctx, cpuctx, event_type, task);
2156 2157
}

Stephane Eranian's avatar
Stephane Eranian committed
2158 2159
static void perf_event_context_sched_in(struct perf_event_context *ctx,
					struct task_struct *task)
2160
{
2161
	struct perf_cpu_context *cpuctx;
2162

2163
	cpuctx = __get_cpu_context(ctx);
2164 2165 2166
	if (cpuctx->task_ctx == ctx)
		return;

2167
	perf_ctx_lock(cpuctx, ctx);
Peter Zijlstra's avatar
Peter Zijlstra committed
2168
	perf_pmu_disable(ctx->pmu);
2169 2170 2171 2172 2173 2174 2175
	/*
	 * We want to keep the following priority order:
	 * cpu pinned (that don't need to move), task pinned,
	 * cpu flexible, task flexible.
	 */
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);

2176
	perf_event_sched_in(cpuctx, ctx, task);
2177 2178

	cpuctx->task_ctx = ctx;
2179

2180 2181 2182
	perf_pmu_enable(ctx->pmu);
	perf_ctx_unlock(cpuctx, ctx);

2183 2184 2185 2186
	/*
	 * Since these rotations are per-cpu, we need to ensure the
	 * cpu-context we got scheduled on is actually rotating.
	 */
2187
	perf_pmu_rotate_start(ctx->pmu);
2188 2189
}

2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200
/*
 * Called from scheduler to add the events of the current task
 * with interrupts disabled.
 *
 * We restore the event value and then enable it.
 *
 * This does not protect us against NMI, but enable()
 * sets the enabled bit in the control field of event _before_
 * accessing the event control register. If a NMI hits, then it will
 * keep the event running.
 */
2201 2202
void __perf_event_task_sched_in(struct task_struct *prev,
				struct task_struct *task)
2203 2204 2205 2206 2207 2208 2209 2210 2211
{
	struct perf_event_context *ctx;
	int ctxn;

	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (likely(!ctx))
			continue;

Stephane Eranian's avatar
Stephane Eranian committed
2212
		perf_event_context_sched_in(ctx, task);
2213
	}
Stephane Eranian's avatar
Stephane Eranian committed
2214 2215 2216 2217 2218 2219
	/*
	 * if cgroup events exist on this CPU, then we need
	 * to check if we have to switch in PMU state.
	 * cgroup event are system-wide mode only
	 */
	if (atomic_read(&__get_cpu_var(perf_cgroup_events)))
2220
		perf_cgroup_sched_in(prev, task);
2221 2222
}

2223 2224 2225 2226 2227 2228 2229 2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249
static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
{
	u64 frequency = event->attr.sample_freq;
	u64 sec = NSEC_PER_SEC;
	u64 divisor, dividend;

	int count_fls, nsec_fls, frequency_fls, sec_fls;

	count_fls = fls64(count);
	nsec_fls = fls64(nsec);
	frequency_fls = fls64(frequency);
	sec_fls = 30;

	/*
	 * We got @count in @nsec, with a target of sample_freq HZ
	 * the target period becomes:
	 *
	 *             @count * 10^9
	 * period = -------------------
	 *          @nsec * sample_freq
	 *
	 */

	/*
	 * Reduce accuracy by one bit such that @a and @b converge
	 * to a similar magnitude.
	 */
2250
#define REDUCE_FLS(a, b)		\
2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289
do {					\
	if (a##_fls > b##_fls) {	\
		a >>= 1;		\
		a##_fls--;		\
	} else {			\
		b >>= 1;		\
		b##_fls--;		\
	}				\
} while (0)

	/*
	 * Reduce accuracy until either term fits in a u64, then proceed with
	 * the other, so that finally we can do a u64/u64 division.
	 */
	while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
		REDUCE_FLS(nsec, frequency);
		REDUCE_FLS(sec, count);
	}

	if (count_fls + sec_fls > 64) {
		divisor = nsec * frequency;

		while (count_fls + sec_fls > 64) {
			REDUCE_FLS(count, sec);
			divisor >>= 1;
		}

		dividend = count * sec;
	} else {
		dividend = count * sec;

		while (nsec_fls + frequency_fls > 64) {
			REDUCE_FLS(nsec, frequency);
			dividend >>= 1;
		}

		divisor = nsec * frequency;
	}

2290 2291 2292
	if (!divisor)
		return dividend;

2293 2294 2295 2296
	return div64_u64(dividend, divisor);
}

static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count)
2297
{
2298
	struct hw_perf_event *hwc = &event->hw;
2299
	s64 period, sample_period;
2300 2301
	s64 delta;

2302
	period = perf_calculate_period(event, nsec, count);
2303 2304 2305 2306 2307 2308 2309 2310 2311 2312

	delta = (s64)(period - hwc->sample_period);
	delta = (delta + 7) / 8; /* low pass filter */

	sample_period = hwc->sample_period + delta;

	if (!sample_period)
		sample_period = 1;

	hwc->sample_period = sample_period;
2313

2314
	if (local64_read(&hwc->period_left) > 8*sample_period) {
2315
		event->pmu->stop(event, PERF_EF_UPDATE);
2316
		local64_set(&hwc->period_left, 0);
2317
		event->pmu->start(event, PERF_EF_RELOAD);
2318
	}
2319 2320
}

2321
static void perf_ctx_adjust_freq(struct perf_event_context *ctx, u64 period)
2322
{
2323 2324
	struct perf_event *event;
	struct hw_perf_event *hwc;
2325 2326
	u64 interrupts, now;
	s64 delta;
2327

2328
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2329
		if (event->state != PERF_EVENT_STATE_ACTIVE)
2330 2331
			continue;

2332
		if (!event_filter_match(event))
2333 2334
			continue;

2335
		hwc = &event->hw;
2336 2337 2338

		interrupts = hwc->interrupts;
		hwc->interrupts = 0;
2339

2340
		/*
2341
		 * unthrottle events on the tick
2342
		 */
2343
		if (interrupts == MAX_INTERRUPTS) {
2344
			perf_log_throttle(event, 1);
2345
			event->pmu->start(event, 0);
2346 2347
		}

2348
		if (!event->attr.freq || !event->attr.sample_freq)
2349 2350
			continue;

2351
		event->pmu->read(event);
2352
		now = local64_read(&event->count);
2353 2354
		delta = now - hwc->freq_count_stamp;
		hwc->freq_count_stamp = now;
2355

2356
		if (delta > 0)
2357
			perf_adjust_period(event, period, delta);
2358 2359 2360
	}
}

2361
/*
2362
 * Round-robin a context's events:
2363
 */
2364
static void rotate_ctx(struct perf_event_context *ctx)
2365
{
2366 2367 2368 2369 2370 2371
	/*
	 * Rotate the first entry last of non-pinned groups. Rotation might be
	 * disabled by the inheritance code.
	 */
	if (!ctx->rotate_disable)
		list_rotate_left(&ctx->flexible_groups);
2372 2373
}

2374
/*
2375 2376 2377
 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
 * because they're strictly cpu affine and rotate_start is called with IRQs
 * disabled, while rotate_context is called from IRQ context.
2378
 */
2379
static void perf_rotate_context(struct perf_cpu_context *cpuctx)
2380
{
2381
	u64 interval = (u64)cpuctx->jiffies_interval * TICK_NSEC;
2382
	struct perf_event_context *ctx = NULL;
2383
	int rotate = 0, remove = 1;
2384

2385
	if (cpuctx->ctx.nr_events) {
2386
		remove = 0;
2387 2388 2389
		if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
			rotate = 1;
	}
2390

2391
	ctx = cpuctx->task_ctx;
2392
	if (ctx && ctx->nr_events) {
2393
		remove = 0;
2394 2395 2396
		if (ctx->nr_events != ctx->nr_active)
			rotate = 1;
	}
2397

2398
	perf_ctx_lock(cpuctx, cpuctx->task_ctx);
Peter Zijlstra's avatar
Peter Zijlstra committed
2399
	perf_pmu_disable(cpuctx->ctx.pmu);
2400
	perf_ctx_adjust_freq(&cpuctx->ctx, interval);
2401
	if (ctx)
2402
		perf_ctx_adjust_freq(ctx, interval);
2403

2404
	if (!rotate)
2405
		goto done;
2406

2407
	cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2408
	if (ctx)
2409
		ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2410

2411
	rotate_ctx(&cpuctx->ctx);
2412 2413
	if (ctx)
		rotate_ctx(ctx);
2414

2415
	perf_event_sched_in(cpuctx, ctx, current);
2416 2417

done:
2418 2419 2420
	if (remove)
		list_del_init(&cpuctx->rotation_list);

Peter Zijlstra's avatar
Peter Zijlstra committed
2421
	perf_pmu_enable(cpuctx->ctx.pmu);
2422
	perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2423 2424 2425 2426 2427 2428
}

void perf_event_task_tick(void)
{
	struct list_head *head = &__get_cpu_var(rotation_list);
	struct perf_cpu_context *cpuctx, *tmp;
2429

2430 2431 2432 2433 2434 2435 2436
	WARN_ON(!irqs_disabled());

	list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
		if (cpuctx->jiffies_interval == 1 ||
				!(jiffies % cpuctx->jiffies_interval))
			perf_rotate_context(cpuctx);
	}
2437 2438
}

2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453
static int event_enable_on_exec(struct perf_event *event,
				struct perf_event_context *ctx)
{
	if (!event->attr.enable_on_exec)
		return 0;

	event->attr.enable_on_exec = 0;
	if (event->state >= PERF_EVENT_STATE_INACTIVE)
		return 0;

	__perf_event_mark_enabled(event, ctx);

	return 1;
}

2454
/*
2455
 * Enable all of a task's events that have been marked enable-on-exec.
2456 2457
 * This expects task == current.
 */
2458
static void perf_event_enable_on_exec(struct perf_event_context *ctx)
2459
{
2460
	struct perf_event *event;
2461 2462
	unsigned long flags;
	int enabled = 0;
2463
	int ret;
2464 2465

	local_irq_save(flags);
2466
	if (!ctx || !ctx->nr_events)
2467 2468
		goto out;

2469 2470 2471 2472 2473 2474 2475
	/*
	 * We must ctxsw out cgroup events to avoid conflict
	 * when invoking perf_task_event_sched_in() later on
	 * in this function. Otherwise we end up trying to
	 * ctxswin cgroup events which are already scheduled
	 * in.
	 */
2476
	perf_cgroup_sched_out(current, NULL);
2477

2478
	raw_spin_lock(&ctx->lock);
2479
	task_ctx_sched_out(ctx);
2480

2481 2482 2483 2484 2485 2486 2487 2488 2489 2490
	list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
	}

	list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
		ret = event_enable_on_exec(event, ctx);
		if (ret)
			enabled = 1;
2491 2492 2493
	}

	/*
2494
	 * Unclone this context if we enabled any event.
2495
	 */
2496 2497
	if (enabled)
		unclone_ctx(ctx);
2498

2499
	raw_spin_unlock(&ctx->lock);
2500

2501 2502 2503
	/*
	 * Also calls ctxswin for cgroup events, if any:
	 */
Stephane Eranian's avatar
Stephane Eranian committed
2504
	perf_event_context_sched_in(ctx, ctx->task);
Peter Zijlstra's avatar
Peter Zijlstra committed
2505
out:
2506 2507 2508
	local_irq_restore(flags);
}

2509
/*
2510
 * Cross CPU call to read the hardware event
2511
 */
2512
static void __perf_event_read(void *info)
2513
{
2514 2515
	struct perf_event *event = info;
	struct perf_event_context *ctx = event->ctx;
2516
	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
Ingo Molnar's avatar
Ingo Molnar committed
2517

2518 2519 2520 2521
	/*
	 * If this is a task context, we need to check whether it is
	 * the current task context of this cpu.  If not it has been
	 * scheduled out before the smp call arrived.  In that case
2522 2523
	 * event->count would have been updated to a recent sample
	 * when the event was scheduled out.
2524 2525 2526 2527
	 */
	if (ctx->task && cpuctx->task_ctx != ctx)
		return;

2528
	raw_spin_lock(&ctx->lock);
Stephane Eranian's avatar
Stephane Eranian committed
2529
	if (ctx->is_active) {
2530
		update_context_time(ctx);
Stephane Eranian's avatar
Stephane Eranian committed
2531 2532
		update_cgrp_time_from_event(event);
	}
2533
	update_event_times(event);
2534 2535
	if (event->state == PERF_EVENT_STATE_ACTIVE)
		event->pmu->read(event);
2536
	raw_spin_unlock(&ctx->lock);
2537 2538
}

2539 2540
static inline u64 perf_event_count(struct perf_event *event)
{
2541
	return local64_read(&event->count) + atomic64_read(&event->child_count);
2542 2543
}

2544
static u64 perf_event_read(struct perf_event *event)
2545 2546
{
	/*
2547 2548
	 * If event is enabled and currently active on a CPU, update the
	 * value in the event structure:
2549
	 */
2550 2551 2552 2553
	if (event->state == PERF_EVENT_STATE_ACTIVE) {
		smp_call_function_single(event->oncpu,
					 __perf_event_read, event, 1);
	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
Peter Zijlstra's avatar
Peter Zijlstra committed
2554 2555 2556
		struct perf_event_context *ctx = event->ctx;
		unsigned long flags;

2557
		raw_spin_lock_irqsave(&ctx->lock, flags);
2558 2559 2560 2561 2562
		/*
		 * may read while context is not active
		 * (e.g., thread is blocked), in that case
		 * we cannot update context time
		 */
Stephane Eranian's avatar
Stephane Eranian committed
2563
		if (ctx->is_active) {
2564
			update_context_time(ctx);
Stephane Eranian's avatar
Stephane Eranian committed
2565 2566
			update_cgrp_time_from_event(event);
		}
2567
		update_event_times(event);
2568
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
2569 2570
	}

2571
	return perf_event_count(event);
2572 2573
}

2574
/*
2575
 * Initialize the perf_event context in a task_struct:
2576
 */
2577
static void __perf_event_init_context(struct perf_event_context *ctx)
2578
{
2579
	raw_spin_lock_init(&ctx->lock);
2580
	mutex_init(&ctx->mutex);
2581 2582
	INIT_LIST_HEAD(&ctx->pinned_groups);
	INIT_LIST_HEAD(&ctx->flexible_groups);
2583 2584
	INIT_LIST_HEAD(&ctx->event_list);
	atomic_set(&ctx->refcount, 1);
2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599
}

static struct perf_event_context *
alloc_perf_context(struct pmu *pmu, struct task_struct *task)
{
	struct perf_event_context *ctx;

	ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
	if (!ctx)
		return NULL;

	__perf_event_init_context(ctx);
	if (task) {
		ctx->task = task;
		get_task_struct(task);
2600
	}
2601 2602 2603
	ctx->pmu = pmu;

	return ctx;
2604 2605
}

2606 2607 2608 2609 2610
static struct task_struct *
find_lively_task_by_vpid(pid_t vpid)
{
	struct task_struct *task;
	int err;
2611 2612

	rcu_read_lock();
2613
	if (!vpid)
2614 2615
		task = current;
	else
2616
		task = find_task_by_vpid(vpid);
2617 2618 2619 2620 2621 2622 2623 2624
	if (task)
		get_task_struct(task);
	rcu_read_unlock();

	if (!task)
		return ERR_PTR(-ESRCH);

	/* Reuse ptrace permission checks for now. */
2625 2626 2627 2628
	err = -EACCES;
	if (!ptrace_may_access(task, PTRACE_MODE_READ))
		goto errout;

2629 2630 2631 2632 2633 2634 2635
	return task;
errout:
	put_task_struct(task);
	return ERR_PTR(err);

}

2636 2637 2638
/*
 * Returns a matching context with refcount and pincount.
 */
2639
static struct perf_event_context *
2640
find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
2641
{
2642
	struct perf_event_context *ctx;
2643
	struct perf_cpu_context *cpuctx;
2644
	unsigned long flags;
2645
	int ctxn, err;
2646

2647
	if (!task) {
2648
		/* Must be root to operate on a CPU event: */
2649
		if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
2650 2651 2652
			return ERR_PTR(-EACCES);

		/*
2653
		 * We could be clever and allow to attach a event to an
2654 2655 2656
		 * offline CPU and activate it when the CPU comes up, but
		 * that's for later.
		 */
2657
		if (!cpu_online(cpu))
2658 2659
			return ERR_PTR(-ENODEV);

2660
		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
2661
		ctx = &cpuctx->ctx;
2662
		get_ctx(ctx);
2663
		++ctx->pin_count;
2664 2665 2666 2667

		return ctx;
	}

2668 2669 2670 2671 2672
	err = -EINVAL;
	ctxn = pmu->task_ctx_nr;
	if (ctxn < 0)
		goto errout;

Peter Zijlstra's avatar
Peter Zijlstra committed
2673
retry:
2674
	ctx = perf_lock_task_context(task, ctxn, &flags);
2675
	if (ctx) {
2676
		unclone_ctx(ctx);
2677
		++ctx->pin_count;
2678
		raw_spin_unlock_irqrestore(&ctx->lock, flags);
2679
	} else {
2680
		ctx = alloc_perf_context(pmu, task);
2681 2682 2683
		err = -ENOMEM;
		if (!ctx)
			goto errout;
2684

2685 2686 2687 2688 2689 2690 2691 2692 2693 2694
		err = 0;
		mutex_lock(&task->perf_event_mutex);
		/*
		 * If it has already passed perf_event_exit_task().
		 * we must see PF_EXITING, it takes this mutex too.
		 */
		if (task->flags & PF_EXITING)
			err = -ESRCH;
		else if (task->perf_event_ctxp[ctxn])
			err = -EAGAIN;
2695
		else {
2696
			get_ctx(ctx);
2697
			++ctx->pin_count;
2698
			rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
2699
		}
2700 2701 2702
		mutex_unlock(&task->perf_event_mutex);

		if (unlikely(err)) {
2703
			put_ctx(ctx);
2704 2705 2706 2707

			if (err == -EAGAIN)
				goto retry;
			goto errout;
2708 2709 2710
		}
	}

2711
	return ctx;
2712

Peter Zijlstra's avatar
Peter Zijlstra committed
2713
errout:
2714
	return ERR_PTR(err);
2715 2716
}

2717 2718
static void perf_event_free_filter(struct perf_event *event);

2719
static void free_event_rcu(struct rcu_head *head)
2720
{
2721
	struct perf_event *event;
2722

2723 2724 2725
	event = container_of(head, struct perf_event, rcu_head);
	if (event->ns)
		put_pid_ns(event->ns);
2726
	perf_event_free_filter(event);
2727
	kfree(event);
2728 2729
}

2730
static void ring_buffer_put(struct ring_buffer *rb);
2731

2732
static void free_event(struct perf_event *event)
2733
{
2734
	irq_work_sync(&event->pending);
2735

2736
	if (!event->parent) {
2737
		if (event->attach_state & PERF_ATTACH_TASK)
Stephane Eranian's avatar
Stephane Eranian committed
2738
			jump_label_dec(&perf_sched_events);
2739
		if (event->attr.mmap || event->attr.mmap_data)
2740 2741 2742 2743 2744
			atomic_dec(&nr_mmap_events);
		if (event->attr.comm)
			atomic_dec(&nr_comm_events);
		if (event->attr.task)
			atomic_dec(&nr_task_events);
2745 2746
		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
			put_callchain_buffers();
2747 2748 2749 2750
		if (is_cgroup_event(event)) {
			atomic_dec(&per_cpu(perf_cgroup_events, event->cpu));
			jump_label_dec(&perf_sched_events);
		}
2751
	}
2752

2753 2754 2755
	if (event->rb) {
		ring_buffer_put(event->rb);
		event->rb = NULL;
2756 2757
	}

Stephane Eranian's avatar
Stephane Eranian committed
2758 2759 2760
	if (is_cgroup_event(event))
		perf_detach_cgroup(event);

2761 2762
	if (event->destroy)
		event->destroy(event);
2763

Peter Zijlstra's avatar
Peter Zijlstra committed
2764 2765 2766
	if (event->ctx)
		put_ctx(event->ctx);

2767
	call_rcu(&event->rcu_head, free_event_rcu);
2768 2769
}

2770
int perf_event_release_kernel(struct perf_event *event)
2771
{
2772
	struct perf_event_context *ctx = event->ctx;
2773

2774
	WARN_ON_ONCE(ctx->parent_ctx);
2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787
	/*
	 * There are two ways this annotation is useful:
	 *
	 *  1) there is a lock recursion from perf_event_exit_task
	 *     see the comment there.
	 *
	 *  2) there is a lock-inversion with mmap_sem through
	 *     perf_event_read_group(), which takes faults while
	 *     holding ctx->mutex, however this is called after
	 *     the last filedesc died, so there is no possibility
	 *     to trigger the AB-BA case.
	 */
	mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
2788
	raw_spin_lock_irq(&ctx->lock);
2789
	perf_group_detach(event);
2790
	raw_spin_unlock_irq(&ctx->lock);
2791
	perf_remove_from_context(event);
2792
	mutex_unlock(&ctx->mutex);
2793

2794
	free_event(event);
2795 2796 2797

	return 0;
}
2798
EXPORT_SYMBOL_GPL(perf_event_release_kernel);
2799

2800 2801 2802 2803
/*
 * Called when the last reference to the file is gone.
 */
static int perf_release(struct inode *inode, struct file *file)
2804
{
2805
	struct perf_event *event = file->private_data;
2806
	struct task_struct *owner;
2807

2808
	file->private_data = NULL;
2809

2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842
	rcu_read_lock();
	owner = ACCESS_ONCE(event->owner);
	/*
	 * Matches the smp_wmb() in perf_event_exit_task(). If we observe
	 * !owner it means the list deletion is complete and we can indeed
	 * free this event, otherwise we need to serialize on
	 * owner->perf_event_mutex.
	 */
	smp_read_barrier_depends();
	if (owner) {
		/*
		 * Since delayed_put_task_struct() also drops the last
		 * task reference we can safely take a new reference
		 * while holding the rcu_read_lock().
		 */
		get_task_struct(owner);
	}
	rcu_read_unlock();

	if (owner) {
		mutex_lock(&owner->perf_event_mutex);
		/*
		 * We have to re-check the event->owner field, if it is cleared
		 * we raced with perf_event_exit_task(), acquiring the mutex
		 * ensured they're done, and we can proceed with freeing the
		 * event.
		 */
		if (event->owner)
			list_del_init(&event->owner_entry);
		mutex_unlock(&owner->perf_event_mutex);
		put_task_struct(owner);
	}

2843
	return perf_event_release_kernel(event);
2844 2845
}

2846
u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
2847
{
2848
	struct perf_event *child;
2849 2850
	u64 total = 0;

2851 2852 2853
	*enabled = 0;
	*running = 0;

2854
	mutex_lock(&event->child_mutex);
2855
	total += perf_event_read(event);
2856 2857 2858 2859 2860 2861
	*enabled += event->total_time_enabled +
			atomic64_read(&event->child_total_time_enabled);
	*running += event->total_time_running +
			atomic64_read(&event->child_total_time_running);

	list_for_each_entry(child, &event->child_list, child_list) {
2862
		total += perf_event_read(child);
2863 2864 2865
		*enabled += child->total_time_enabled;
		*running += child->total_time_running;
	}
2866
	mutex_unlock(&event->child_mutex);
2867 2868 2869

	return total;
}
2870
EXPORT_SYMBOL_GPL(perf_event_read_value);
2871

2872
static int perf_event_read_group(struct perf_event *event,
2873 2874
				   u64 read_format, char __user *buf)
{
2875
	struct perf_event *leader = event->group_leader, *sub;
2876 2877
	int n = 0, size = 0, ret = -EFAULT;
	struct perf_event_context *ctx = leader->ctx;
2878
	u64 values[5];
2879
	u64 count, enabled, running;
2880

2881
	mutex_lock(&ctx->mutex);
2882
	count = perf_event_read_value(leader, &enabled, &running);
2883 2884

	values[n++] = 1 + leader->nr_siblings;
2885 2886 2887 2888
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
2889 2890 2891
	values[n++] = count;
	if (read_format & PERF_FORMAT_ID)
		values[n++] = primary_event_id(leader);
2892 2893 2894 2895

	size = n * sizeof(u64);

	if (copy_to_user(buf, values, size))
2896
		goto unlock;
2897

2898
	ret = size;
2899

2900
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
2901
		n = 0;
2902

2903
		values[n++] = perf_event_read_value(sub, &enabled, &running);
2904 2905 2906 2907 2908
		if (read_format & PERF_FORMAT_ID)
			values[n++] = primary_event_id(sub);

		size = n * sizeof(u64);

2909
		if (copy_to_user(buf + ret, values, size)) {
2910 2911 2912
			ret = -EFAULT;
			goto unlock;
		}
2913 2914

		ret += size;
2915
	}
2916 2917
unlock:
	mutex_unlock(&ctx->mutex);
2918

2919
	return ret;
2920 2921
}

2922
static int perf_event_read_one(struct perf_event *event,
2923 2924
				 u64 read_format, char __user *buf)
{
2925
	u64 enabled, running;
2926 2927 2928
	u64 values[4];
	int n = 0;

2929 2930 2931 2932 2933
	values[n++] = perf_event_read_value(event, &enabled, &running);
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
		values[n++] = enabled;
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
		values[n++] = running;
2934
	if (read_format & PERF_FORMAT_ID)
2935
		values[n++] = primary_event_id(event);
2936 2937 2938 2939 2940 2941 2942

	if (copy_to_user(buf, values, n * sizeof(u64)))
		return -EFAULT;

	return n * sizeof(u64);
}

2943
/*
2944
 * Read the performance event - simple non blocking version for now
2945 2946
 */
static ssize_t
2947
perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
2948
{
2949
	u64 read_format = event->attr.read_format;
2950
	int ret;
2951

2952
	/*
2953
	 * Return end-of-file for a read on a event that is in
2954 2955 2956
	 * error state (i.e. because it was pinned but it couldn't be
	 * scheduled on to the CPU at some point).
	 */
2957
	if (event->state == PERF_EVENT_STATE_ERROR)
2958 2959
		return 0;

2960
	if (count < event->read_size)
2961 2962
		return -ENOSPC;

2963
	WARN_ON_ONCE(event->ctx->parent_ctx);
2964
	if (read_format & PERF_FORMAT_GROUP)
2965
		ret = perf_event_read_group(event, read_format, buf);
2966
	else
2967
		ret = perf_event_read_one(event, read_format, buf);
2968

2969
	return ret;
2970 2971 2972 2973 2974
}

static ssize_t
perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
{
2975
	struct perf_event *event = file->private_data;
2976

2977
	return perf_read_hw(event, buf, count);
2978 2979 2980 2981
}

static unsigned int perf_poll(struct file *file, poll_table *wait)
{
2982
	struct perf_event *event = file->private_data;
2983
	struct ring_buffer *rb;
2984
	unsigned int events = POLL_HUP;
2985 2986

	rcu_read_lock();
2987 2988 2989
	rb = rcu_dereference(event->rb);
	if (rb)
		events = atomic_xchg(&rb->poll, 0);
2990
	rcu_read_unlock();
2991

2992
	poll_wait(file, &event->waitq, wait);
2993 2994 2995 2996

	return events;
}

2997
static void perf_event_reset(struct perf_event *event)
2998
{
2999
	(void)perf_event_read(event);
3000
	local64_set(&event->count, 0);
3001
	perf_event_update_userpage(event);
3002 3003
}

3004
/*
3005 3006 3007 3008
 * Holding the top-level event's child_mutex means that any
 * descendant process that has inherited this event will block
 * in sync_child_event if it goes to exit, thus satisfying the
 * task existence requirements of perf_event_enable/disable.
3009
 */
3010 3011
static void perf_event_for_each_child(struct perf_event *event,
					void (*func)(struct perf_event *))
3012
{
3013
	struct perf_event *child;
3014

3015 3016 3017 3018
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->child_mutex);
	func(event);
	list_for_each_entry(child, &event->child_list, child_list)
3019
		func(child);
3020
	mutex_unlock(&event->child_mutex);
3021 3022
}

3023 3024
static void perf_event_for_each(struct perf_event *event,
				  void (*func)(struct perf_event *))
3025
{
3026 3027
	struct perf_event_context *ctx = event->ctx;
	struct perf_event *sibling;
3028

3029 3030
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
3031
	event = event->group_leader;
3032

3033 3034 3035 3036
	perf_event_for_each_child(event, func);
	func(event);
	list_for_each_entry(sibling, &event->sibling_list, group_entry)
		perf_event_for_each_child(event, func);
3037
	mutex_unlock(&ctx->mutex);
3038 3039
}

3040
static int perf_event_period(struct perf_event *event, u64 __user *arg)
3041
{
3042
	struct perf_event_context *ctx = event->ctx;
3043 3044 3045
	int ret = 0;
	u64 value;

3046
	if (!is_sampling_event(event))
3047 3048
		return -EINVAL;

3049
	if (copy_from_user(&value, arg, sizeof(value)))
3050 3051 3052 3053 3054
		return -EFAULT;

	if (!value)
		return -EINVAL;

3055
	raw_spin_lock_irq(&ctx->lock);
3056 3057
	if (event->attr.freq) {
		if (value > sysctl_perf_event_sample_rate) {
3058 3059 3060 3061
			ret = -EINVAL;
			goto unlock;
		}

3062
		event->attr.sample_freq = value;
3063
	} else {
3064 3065
		event->attr.sample_period = value;
		event->hw.sample_period = value;
3066 3067
	}
unlock:
3068
	raw_spin_unlock_irq(&ctx->lock);
3069 3070 3071 3072

	return ret;
}

3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093
static const struct file_operations perf_fops;

static struct perf_event *perf_fget_light(int fd, int *fput_needed)
{
	struct file *file;

	file = fget_light(fd, fput_needed);
	if (!file)
		return ERR_PTR(-EBADF);

	if (file->f_op != &perf_fops) {
		fput_light(file, *fput_needed);
		*fput_needed = 0;
		return ERR_PTR(-EBADF);
	}

	return file->private_data;
}

static int perf_event_set_output(struct perf_event *event,
				 struct perf_event *output_event);
3094
static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3095

3096 3097
static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
{
3098 3099
	struct perf_event *event = file->private_data;
	void (*func)(struct perf_event *);
3100
	u32 flags = arg;
3101 3102

	switch (cmd) {
3103 3104
	case PERF_EVENT_IOC_ENABLE:
		func = perf_event_enable;
3105
		break;
3106 3107
	case PERF_EVENT_IOC_DISABLE:
		func = perf_event_disable;
3108
		break;
3109 3110
	case PERF_EVENT_IOC_RESET:
		func = perf_event_reset;
3111
		break;
3112

3113 3114
	case PERF_EVENT_IOC_REFRESH:
		return perf_event_refresh(event, arg);
3115

3116 3117
	case PERF_EVENT_IOC_PERIOD:
		return perf_event_period(event, (u64 __user *)arg);
3118

3119
	case PERF_EVENT_IOC_SET_OUTPUT:
3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136
	{
		struct perf_event *output_event = NULL;
		int fput_needed = 0;
		int ret;

		if (arg != -1) {
			output_event = perf_fget_light(arg, &fput_needed);
			if (IS_ERR(output_event))
				return PTR_ERR(output_event);
		}

		ret = perf_event_set_output(event, output_event);
		if (output_event)
			fput_light(output_event->filp, fput_needed);

		return ret;
	}
3137

3138 3139 3140
	case PERF_EVENT_IOC_SET_FILTER:
		return perf_event_set_filter(event, (void __user *)arg);

3141
	default:
3142
		return -ENOTTY;
3143
	}
3144 3145

	if (flags & PERF_IOC_FLAG_GROUP)
3146
		perf_event_for_each(event, func);
3147
	else
3148
		perf_event_for_each_child(event, func);
3149 3150

	return 0;
3151 3152
}

3153
int perf_event_task_enable(void)
3154
{
3155
	struct perf_event *event;
3156

3157 3158 3159 3160
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_enable);
	mutex_unlock(&current->perf_event_mutex);
3161 3162 3163 3164

	return 0;
}

3165
int perf_event_task_disable(void)
3166
{
3167
	struct perf_event *event;
3168

3169 3170 3171 3172
	mutex_lock(&current->perf_event_mutex);
	list_for_each_entry(event, &current->perf_event_list, owner_entry)
		perf_event_for_each_child(event, perf_event_disable);
	mutex_unlock(&current->perf_event_mutex);
3173 3174 3175 3176

	return 0;
}

3177 3178
#ifndef PERF_EVENT_INDEX_OFFSET
# define PERF_EVENT_INDEX_OFFSET 0
3179 3180
#endif

3181
static int perf_event_index(struct perf_event *event)
3182
{
3183 3184 3185
	if (event->hw.state & PERF_HES_STOPPED)
		return 0;

3186
	if (event->state != PERF_EVENT_STATE_ACTIVE)
3187 3188
		return 0;

3189
	return event->hw.idx + 1 - PERF_EVENT_INDEX_OFFSET;
3190 3191
}

3192
static void calc_timer_values(struct perf_event *event,
3193 3194
				u64 *enabled,
				u64 *running)
3195 3196 3197 3198 3199 3200 3201 3202 3203
{
	u64 now, ctx_time;

	now = perf_clock();
	ctx_time = event->shadow_ctx_time + now;
	*enabled = ctx_time - event->tstamp_enabled;
	*running = ctx_time - event->tstamp_running;
}

3204 3205 3206 3207 3208
/*
 * Callers need to ensure there can be no nesting of this function, otherwise
 * the seqlock logic goes bad. We can not serialize this because the arch
 * code calls this from NMI context.
 */
3209
void perf_event_update_userpage(struct perf_event *event)
3210
{
3211
	struct perf_event_mmap_page *userpg;
3212
	struct ring_buffer *rb;
3213
	u64 enabled, running;
3214 3215

	rcu_read_lock();
3216 3217 3218 3219 3220 3221 3222 3223 3224 3225
	/*
	 * compute total_time_enabled, total_time_running
	 * based on snapshot values taken when the event
	 * was last scheduled in.
	 *
	 * we cannot simply called update_context_time()
	 * because of locking issue as we can be called in
	 * NMI context
	 */
	calc_timer_values(event, &enabled, &running);
3226 3227
	rb = rcu_dereference(event->rb);
	if (!rb)
3228 3229
		goto unlock;

3230
	userpg = rb->user_page;
3231

3232 3233 3234 3235 3236
	/*
	 * Disable preemption so as to not let the corresponding user-space
	 * spin too long if we get preempted.
	 */
	preempt_disable();
3237
	++userpg->lock;
3238
	barrier();
3239
	userpg->index = perf_event_index(event);
3240
	userpg->offset = perf_event_count(event);
3241
	if (event->state == PERF_EVENT_STATE_ACTIVE)
3242
		userpg->offset -= local64_read(&event->hw.prev_count);
3243

3244
	userpg->time_enabled = enabled +
3245
			atomic64_read(&event->child_total_time_enabled);
3246

3247
	userpg->time_running = running +
3248
			atomic64_read(&event->child_total_time_running);
3249

3250
	barrier();
3251
	++userpg->lock;
3252
	preempt_enable();
3253
unlock:
3254
	rcu_read_unlock();
3255 3256
}

3257 3258 3259
static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
{
	struct perf_event *event = vma->vm_file->private_data;
3260
	struct ring_buffer *rb;
3261 3262 3263 3264 3265 3266 3267 3268 3269
	int ret = VM_FAULT_SIGBUS;

	if (vmf->flags & FAULT_FLAG_MKWRITE) {
		if (vmf->pgoff == 0)
			ret = 0;
		return ret;
	}

	rcu_read_lock();
3270 3271
	rb = rcu_dereference(event->rb);
	if (!rb)
3272 3273 3274 3275 3276
		goto unlock;

	if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
		goto unlock;

3277
	vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291
	if (!vmf->page)
		goto unlock;

	get_page(vmf->page);
	vmf->page->mapping = vma->vm_file->f_mapping;
	vmf->page->index   = vmf->pgoff;

	ret = 0;
unlock:
	rcu_read_unlock();

	return ret;
}

3292
static void rb_free_rcu(struct rcu_head *rcu_head)
3293
{
3294
	struct ring_buffer *rb;
3295

3296 3297
	rb = container_of(rcu_head, struct ring_buffer, rcu_head);
	rb_free(rb);
3298 3299
}

3300
static struct ring_buffer *ring_buffer_get(struct perf_event *event)
3301
{
3302
	struct ring_buffer *rb;
3303

3304
	rcu_read_lock();
3305 3306 3307 3308
	rb = rcu_dereference(event->rb);
	if (rb) {
		if (!atomic_inc_not_zero(&rb->refcount))
			rb = NULL;
3309 3310 3311
	}
	rcu_read_unlock();

3312
	return rb;
3313 3314
}

3315
static void ring_buffer_put(struct ring_buffer *rb)
3316
{
3317
	if (!atomic_dec_and_test(&rb->refcount))
3318
		return;
3319

3320
	call_rcu(&rb->rcu_head, rb_free_rcu);
3321 3322 3323 3324
}

static void perf_mmap_open(struct vm_area_struct *vma)
{
3325
	struct perf_event *event = vma->vm_file->private_data;
3326

3327
	atomic_inc(&event->mmap_count);
3328 3329 3330 3331
}

static void perf_mmap_close(struct vm_area_struct *vma)
{
3332
	struct perf_event *event = vma->vm_file->private_data;
3333

3334
	if (atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex)) {
3335
		unsigned long size = perf_data_size(event->rb);
3336
		struct user_struct *user = event->mmap_user;
3337
		struct ring_buffer *rb = event->rb;
3338

3339
		atomic_long_sub((size >> PAGE_SHIFT) + 1, &user->locked_vm);
3340
		vma->vm_mm->pinned_vm -= event->mmap_locked;
3341
		rcu_assign_pointer(event->rb, NULL);
3342
		mutex_unlock(&event->mmap_mutex);
3343

3344
		ring_buffer_put(rb);
3345
		free_uid(user);
3346
	}
3347 3348
}

3349
static const struct vm_operations_struct perf_mmap_vmops = {
3350 3351 3352 3353
	.open		= perf_mmap_open,
	.close		= perf_mmap_close,
	.fault		= perf_mmap_fault,
	.page_mkwrite	= perf_mmap_fault,
3354 3355 3356 3357
};

static int perf_mmap(struct file *file, struct vm_area_struct *vma)
{
3358
	struct perf_event *event = file->private_data;
3359
	unsigned long user_locked, user_lock_limit;
3360
	struct user_struct *user = current_user();
3361
	unsigned long locked, lock_limit;
3362
	struct ring_buffer *rb;
3363 3364
	unsigned long vma_size;
	unsigned long nr_pages;
3365
	long user_extra, extra;
3366
	int ret = 0, flags = 0;
3367

3368 3369 3370
	/*
	 * Don't allow mmap() of inherited per-task counters. This would
	 * create a performance issue due to all children writing to the
3371
	 * same rb.
3372 3373 3374 3375
	 */
	if (event->cpu == -1 && event->attr.inherit)
		return -EINVAL;

3376
	if (!(vma->vm_flags & VM_SHARED))
3377
		return -EINVAL;
3378 3379 3380 3381

	vma_size = vma->vm_end - vma->vm_start;
	nr_pages = (vma_size / PAGE_SIZE) - 1;

3382
	/*
3383
	 * If we have rb pages ensure they're a power-of-two number, so we
3384 3385 3386
	 * can do bitmasks instead of modulo.
	 */
	if (nr_pages != 0 && !is_power_of_2(nr_pages))
3387 3388
		return -EINVAL;

3389
	if (vma_size != PAGE_SIZE * (1 + nr_pages))
3390 3391
		return -EINVAL;

3392 3393
	if (vma->vm_pgoff != 0)
		return -EINVAL;
3394

3395 3396
	WARN_ON_ONCE(event->ctx->parent_ctx);
	mutex_lock(&event->mmap_mutex);
3397 3398 3399
	if (event->rb) {
		if (event->rb->nr_pages == nr_pages)
			atomic_inc(&event->rb->refcount);
3400
		else
3401 3402 3403 3404
			ret = -EINVAL;
		goto unlock;
	}

3405
	user_extra = nr_pages + 1;
3406
	user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
3407 3408 3409 3410 3411 3412

	/*
	 * Increase the limit linearly with more CPUs:
	 */
	user_lock_limit *= num_online_cpus();

3413
	user_locked = atomic_long_read(&user->locked_vm) + user_extra;
3414

3415 3416 3417
	extra = 0;
	if (user_locked > user_lock_limit)
		extra = user_locked - user_lock_limit;
3418

3419
	lock_limit = rlimit(RLIMIT_MEMLOCK);
3420
	lock_limit >>= PAGE_SHIFT;
3421
	locked = vma->vm_mm->pinned_vm + extra;
3422

3423 3424
	if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
		!capable(CAP_IPC_LOCK)) {
3425 3426 3427
		ret = -EPERM;
		goto unlock;
	}
3428

3429
	WARN_ON(event->rb);
3430

3431
	if (vma->vm_flags & VM_WRITE)
3432
		flags |= RING_BUFFER_WRITABLE;
3433

3434 3435 3436 3437
	rb = rb_alloc(nr_pages, 
		event->attr.watermark ? event->attr.wakeup_watermark : 0,
		event->cpu, flags);

3438
	if (!rb) {
3439
		ret = -ENOMEM;
3440
		goto unlock;
3441
	}
3442
	rcu_assign_pointer(event->rb, rb);
3443

3444 3445 3446
	atomic_long_add(user_extra, &user->locked_vm);
	event->mmap_locked = extra;
	event->mmap_user = get_current_user();
3447
	vma->vm_mm->pinned_vm += event->mmap_locked;
3448

3449
unlock:
3450 3451
	if (!ret)
		atomic_inc(&event->mmap_count);
3452
	mutex_unlock(&event->mmap_mutex);
3453 3454 3455

	vma->vm_flags |= VM_RESERVED;
	vma->vm_ops = &perf_mmap_vmops;
3456 3457

	return ret;
3458 3459
}

Peter Zijlstra's avatar
Peter Zijlstra committed
3460 3461 3462
static int perf_fasync(int fd, struct file *filp, int on)
{
	struct inode *inode = filp->f_path.dentry->d_inode;
3463
	struct perf_event *event = filp->private_data;
Peter Zijlstra's avatar
Peter Zijlstra committed
3464 3465 3466
	int retval;

	mutex_lock(&inode->i_mutex);
3467
	retval = fasync_helper(fd, filp, on, &event->fasync);
Peter Zijlstra's avatar
Peter Zijlstra committed
3468 3469 3470 3471 3472 3473 3474 3475
	mutex_unlock(&inode->i_mutex);

	if (retval < 0)
		return retval;

	return 0;
}

3476
static const struct file_operations perf_fops = {
3477
	.llseek			= no_llseek,
3478 3479 3480
	.release		= perf_release,
	.read			= perf_read,
	.poll			= perf_poll,
3481 3482
	.unlocked_ioctl		= perf_ioctl,
	.compat_ioctl		= perf_ioctl,
3483
	.mmap			= perf_mmap,
Peter Zijlstra's avatar
Peter Zijlstra committed
3484
	.fasync			= perf_fasync,
3485 3486
};

3487
/*
3488
 * Perf event wakeup
3489 3490 3491 3492 3493
 *
 * If there's data, ensure we set the poll() state and publish everything
 * to user-space before waking everybody up.
 */

3494
void perf_event_wakeup(struct perf_event *event)
3495
{
3496
	wake_up_all(&event->waitq);
3497

3498 3499 3500
	if (event->pending_kill) {
		kill_fasync(&event->fasync, SIGIO, event->pending_kill);
		event->pending_kill = 0;
3501
	}
3502 3503
}

3504
static void perf_pending_event(struct irq_work *entry)
3505
{
3506 3507
	struct perf_event *event = container_of(entry,
			struct perf_event, pending);
3508

3509 3510 3511
	if (event->pending_disable) {
		event->pending_disable = 0;
		__perf_event_disable(event);
3512 3513
	}

3514 3515 3516
	if (event->pending_wakeup) {
		event->pending_wakeup = 0;
		perf_event_wakeup(event);
3517 3518 3519
	}
}

3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538 3539 3540
/*
 * We assume there is only KVM supporting the callbacks.
 * Later on, we might change it to a list if there is
 * another virtualization implementation supporting the callbacks.
 */
struct perf_guest_info_callbacks *perf_guest_cbs;

int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
	perf_guest_cbs = cbs;
	return 0;
}
EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);

int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
{
	perf_guest_cbs = NULL;
	return 0;
}
EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);

3541 3542 3543
static void __perf_event_header__init_id(struct perf_event_header *header,
					 struct perf_sample_data *data,
					 struct perf_event *event)
3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570
{
	u64 sample_type = event->attr.sample_type;

	data->type = sample_type;
	header->size += event->id_header_size;

	if (sample_type & PERF_SAMPLE_TID) {
		/* namespace issues */
		data->tid_entry.pid = perf_event_pid(event, current);
		data->tid_entry.tid = perf_event_tid(event, current);
	}

	if (sample_type & PERF_SAMPLE_TIME)
		data->time = perf_clock();

	if (sample_type & PERF_SAMPLE_ID)
		data->id = primary_event_id(event);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		data->stream_id = event->id;

	if (sample_type & PERF_SAMPLE_CPU) {
		data->cpu_entry.cpu	 = raw_smp_processor_id();
		data->cpu_entry.reserved = 0;
	}
}

3571 3572 3573
void perf_event_header__init_id(struct perf_event_header *header,
				struct perf_sample_data *data,
				struct perf_event *event)
3574 3575 3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599
{
	if (event->attr.sample_id_all)
		__perf_event_header__init_id(header, data, event);
}

static void __perf_event__output_id_sample(struct perf_output_handle *handle,
					   struct perf_sample_data *data)
{
	u64 sample_type = data->type;

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);
}

3600 3601 3602
void perf_event__output_id_sample(struct perf_event *event,
				  struct perf_output_handle *handle,
				  struct perf_sample_data *sample)
3603 3604 3605 3606 3607
{
	if (event->attr.sample_id_all)
		__perf_event__output_id_sample(handle, sample);
}

3608
static void perf_output_read_one(struct perf_output_handle *handle,
3609 3610
				 struct perf_event *event,
				 u64 enabled, u64 running)
3611
{
3612
	u64 read_format = event->attr.read_format;
3613 3614 3615
	u64 values[4];
	int n = 0;

3616
	values[n++] = perf_event_count(event);
3617
	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
3618
		values[n++] = enabled +
3619
			atomic64_read(&event->child_total_time_enabled);
3620 3621
	}
	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
3622
		values[n++] = running +
3623
			atomic64_read(&event->child_total_time_running);
3624 3625
	}
	if (read_format & PERF_FORMAT_ID)
3626
		values[n++] = primary_event_id(event);
3627

3628
	__output_copy(handle, values, n * sizeof(u64));
3629 3630 3631
}

/*
3632
 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
3633 3634
 */
static void perf_output_read_group(struct perf_output_handle *handle,
3635 3636
			    struct perf_event *event,
			    u64 enabled, u64 running)
3637
{
3638 3639
	struct perf_event *leader = event->group_leader, *sub;
	u64 read_format = event->attr.read_format;
3640 3641 3642 3643 3644 3645
	u64 values[5];
	int n = 0;

	values[n++] = 1 + leader->nr_siblings;

	if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3646
		values[n++] = enabled;
3647 3648

	if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3649
		values[n++] = running;
3650

3651
	if (leader != event)
3652 3653
		leader->pmu->read(leader);

3654
	values[n++] = perf_event_count(leader);
3655
	if (read_format & PERF_FORMAT_ID)
3656
		values[n++] = primary_event_id(leader);
3657

3658
	__output_copy(handle, values, n * sizeof(u64));
3659

3660
	list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3661 3662
		n = 0;

3663
		if (sub != event)
3664 3665
			sub->pmu->read(sub);

3666
		values[n++] = perf_event_count(sub);
3667
		if (read_format & PERF_FORMAT_ID)
3668
			values[n++] = primary_event_id(sub);
3669

3670
		__output_copy(handle, values, n * sizeof(u64));
3671 3672 3673
	}
}

3674 3675 3676
#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
				 PERF_FORMAT_TOTAL_TIME_RUNNING)

3677
static void perf_output_read(struct perf_output_handle *handle,
3678
			     struct perf_event *event)
3679
{
3680
	u64 enabled = 0, running = 0;
3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691
	u64 read_format = event->attr.read_format;

	/*
	 * compute total_time_enabled, total_time_running
	 * based on snapshot values taken when the event
	 * was last scheduled in.
	 *
	 * we cannot simply called update_context_time()
	 * because of locking issue as we are called in
	 * NMI context
	 */
3692 3693
	if (read_format & PERF_FORMAT_TOTAL_TIMES)
		calc_timer_values(event, &enabled, &running);
3694

3695
	if (event->attr.read_format & PERF_FORMAT_GROUP)
3696
		perf_output_read_group(handle, event, enabled, running);
3697
	else
3698
		perf_output_read_one(handle, event, enabled, running);
3699 3700
}

3701 3702 3703
void perf_output_sample(struct perf_output_handle *handle,
			struct perf_event_header *header,
			struct perf_sample_data *data,
3704
			struct perf_event *event)
3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734
{
	u64 sample_type = data->type;

	perf_output_put(handle, *header);

	if (sample_type & PERF_SAMPLE_IP)
		perf_output_put(handle, data->ip);

	if (sample_type & PERF_SAMPLE_TID)
		perf_output_put(handle, data->tid_entry);

	if (sample_type & PERF_SAMPLE_TIME)
		perf_output_put(handle, data->time);

	if (sample_type & PERF_SAMPLE_ADDR)
		perf_output_put(handle, data->addr);

	if (sample_type & PERF_SAMPLE_ID)
		perf_output_put(handle, data->id);

	if (sample_type & PERF_SAMPLE_STREAM_ID)
		perf_output_put(handle, data->stream_id);

	if (sample_type & PERF_SAMPLE_CPU)
		perf_output_put(handle, data->cpu_entry);

	if (sample_type & PERF_SAMPLE_PERIOD)
		perf_output_put(handle, data->period);

	if (sample_type & PERF_SAMPLE_READ)
3735
		perf_output_read(handle, event);
3736 3737 3738 3739 3740 3741 3742 3743 3744 3745

	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
		if (data->callchain) {
			int size = 1;

			if (data->callchain)
				size += data->callchain->nr;

			size *= sizeof(u64);

3746
			__output_copy(handle, data->callchain, size);
3747 3748 3749 3750 3751 3752 3753 3754 3755
		} else {
			u64 nr = 0;
			perf_output_put(handle, nr);
		}
	}

	if (sample_type & PERF_SAMPLE_RAW) {
		if (data->raw) {
			perf_output_put(handle, data->raw->size);
3756 3757
			__output_copy(handle, data->raw->data,
					   data->raw->size);
3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768
		} else {
			struct {
				u32	size;
				u32	data;
			} raw = {
				.size = sizeof(u32),
				.data = 0,
			};
			perf_output_put(handle, raw);
		}
	}
3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782

	if (!event->attr.watermark) {
		int wakeup_events = event->attr.wakeup_events;

		if (wakeup_events) {
			struct ring_buffer *rb = handle->rb;
			int events = local_inc_return(&rb->events);

			if (events >= wakeup_events) {
				local_sub(wakeup_events, &rb->events);
				local_inc(&rb->wakeup);
			}
		}
	}
3783 3784 3785 3786
}

void perf_prepare_sample(struct perf_event_header *header,
			 struct perf_sample_data *data,
3787
			 struct perf_event *event,
3788
			 struct pt_regs *regs)
3789
{
3790
	u64 sample_type = event->attr.sample_type;
3791

3792
	header->type = PERF_RECORD_SAMPLE;
3793
	header->size = sizeof(*header) + event->header_size;
3794 3795 3796

	header->misc = 0;
	header->misc |= perf_misc_flags(regs);
3797

3798
	__perf_event_header__init_id(header, data, event);
3799

3800
	if (sample_type & PERF_SAMPLE_IP)
3801 3802
		data->ip = perf_instruction_pointer(regs);

3803
	if (sample_type & PERF_SAMPLE_CALLCHAIN) {
3804
		int size = 1;
3805

3806 3807 3808 3809 3810 3811
		data->callchain = perf_callchain(regs);

		if (data->callchain)
			size += data->callchain->nr;

		header->size += size * sizeof(u64);
3812 3813
	}

3814
	if (sample_type & PERF_SAMPLE_RAW) {
3815 3816 3817 3818 3819 3820 3821 3822
		int size = sizeof(u32);

		if (data->raw)
			size += data->raw->size;
		else
			size += sizeof(u32);

		WARN_ON_ONCE(size & (sizeof(u64)-1));
3823
		header->size += size;
3824
	}
3825
}
3826

3827
static void perf_event_output(struct perf_event *event,
3828 3829 3830 3831 3832
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
	struct perf_output_handle handle;
	struct perf_event_header header;
3833

3834 3835 3836
	/* protect the callchain buffers */
	rcu_read_lock();

3837
	perf_prepare_sample(&header, data, event, regs);
3838

3839
	if (perf_output_begin(&handle, event, header.size))
3840
		goto exit;
3841

3842
	perf_output_sample(&handle, &header, data, event);
3843

3844
	perf_output_end(&handle);
3845 3846 3847

exit:
	rcu_read_unlock();
3848 3849
}

3850
/*
3851
 * read event_id
3852 3853 3854 3855 3856 3857 3858 3859 3860 3861
 */

struct perf_read_event {
	struct perf_event_header	header;

	u32				pid;
	u32				tid;
};

static void
3862
perf_event_read_event(struct perf_event *event,
3863 3864 3865
			struct task_struct *task)
{
	struct perf_output_handle handle;
3866
	struct perf_sample_data sample;
3867
	struct perf_read_event read_event = {
3868
		.header = {
3869
			.type = PERF_RECORD_READ,
3870
			.misc = 0,
3871
			.size = sizeof(read_event) + event->read_size,
3872
		},
3873 3874
		.pid = perf_event_pid(event, task),
		.tid = perf_event_tid(event, task),
3875
	};
3876
	int ret;
3877

3878
	perf_event_header__init_id(&read_event.header, &sample, event);
3879
	ret = perf_output_begin(&handle, event, read_event.header.size);
3880 3881 3882
	if (ret)
		return;

3883
	perf_output_put(&handle, read_event);
3884
	perf_output_read(&handle, event);
3885
	perf_event__output_id_sample(event, &handle, &sample);
3886

3887 3888 3889
	perf_output_end(&handle);
}

3890
/*
3891 3892
 * task tracking -- fork/exit
 *
3893
 * enabled by: attr.comm | attr.mmap | attr.mmap_data | attr.task
3894 3895
 */

3896
struct perf_task_event {
3897
	struct task_struct		*task;
3898
	struct perf_event_context	*task_ctx;
3899 3900 3901 3902 3903 3904

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				ppid;
3905 3906
		u32				tid;
		u32				ptid;
3907
		u64				time;
3908
	} event_id;
3909 3910
};

3911
static void perf_event_task_output(struct perf_event *event,
3912
				     struct perf_task_event *task_event)
3913 3914
{
	struct perf_output_handle handle;
3915
	struct perf_sample_data	sample;
3916
	struct task_struct *task = task_event->task;
3917
	int ret, size = task_event->event_id.header.size;
3918

3919
	perf_event_header__init_id(&task_event->event_id.header, &sample, event);
3920

3921
	ret = perf_output_begin(&handle, event,
3922
				task_event->event_id.header.size);
3923
	if (ret)
3924
		goto out;
3925

3926 3927
	task_event->event_id.pid = perf_event_pid(event, task);
	task_event->event_id.ppid = perf_event_pid(event, current);
3928

3929 3930
	task_event->event_id.tid = perf_event_tid(event, task);
	task_event->event_id.ptid = perf_event_tid(event, current);
3931

3932
	perf_output_put(&handle, task_event->event_id);
3933

3934 3935
	perf_event__output_id_sample(event, &handle, &sample);

3936
	perf_output_end(&handle);
3937 3938
out:
	task_event->event_id.header.size = size;
3939 3940
}

3941
static int perf_event_task_match(struct perf_event *event)
3942
{
3943
	if (event->state < PERF_EVENT_STATE_INACTIVE)
3944 3945
		return 0;

3946
	if (!event_filter_match(event))
3947 3948
		return 0;

3949 3950
	if (event->attr.comm || event->attr.mmap ||
	    event->attr.mmap_data || event->attr.task)
3951 3952 3953 3954 3955
		return 1;

	return 0;
}

3956
static void perf_event_task_ctx(struct perf_event_context *ctx,
3957
				  struct perf_task_event *task_event)
3958
{
3959
	struct perf_event *event;
3960

3961 3962 3963
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_task_match(event))
			perf_event_task_output(event, task_event);
3964 3965 3966
	}
}

3967
static void perf_event_task_event(struct perf_task_event *task_event)
3968 3969
{
	struct perf_cpu_context *cpuctx;
3970
	struct perf_event_context *ctx;
3971
	struct pmu *pmu;
3972
	int ctxn;
3973

3974
	rcu_read_lock();
3975
	list_for_each_entry_rcu(pmu, &pmus, entry) {
3976
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
3977 3978
		if (cpuctx->active_pmu != pmu)
			goto next;
3979
		perf_event_task_ctx(&cpuctx->ctx, task_event);
3980 3981 3982 3983 3984

		ctx = task_event->task_ctx;
		if (!ctx) {
			ctxn = pmu->task_ctx_nr;
			if (ctxn < 0)
3985
				goto next;
3986 3987 3988 3989
			ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		}
		if (ctx)
			perf_event_task_ctx(ctx, task_event);
3990 3991
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
3992
	}
3993 3994 3995
	rcu_read_unlock();
}

3996 3997
static void perf_event_task(struct task_struct *task,
			      struct perf_event_context *task_ctx,
3998
			      int new)
3999
{
4000
	struct perf_task_event task_event;
4001

4002 4003 4004
	if (!atomic_read(&nr_comm_events) &&
	    !atomic_read(&nr_mmap_events) &&
	    !atomic_read(&nr_task_events))
4005 4006
		return;

4007
	task_event = (struct perf_task_event){
4008 4009
		.task	  = task,
		.task_ctx = task_ctx,
4010
		.event_id    = {
4011
			.header = {
4012
				.type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
4013
				.misc = 0,
4014
				.size = sizeof(task_event.event_id),
4015
			},
4016 4017
			/* .pid  */
			/* .ppid */
4018 4019
			/* .tid  */
			/* .ptid */
4020
			.time = perf_clock(),
4021 4022 4023
		},
	};

4024
	perf_event_task_event(&task_event);
4025 4026
}

4027
void perf_event_fork(struct task_struct *task)
4028
{
4029
	perf_event_task(task, NULL, 1);
4030 4031
}

4032 4033 4034 4035 4036
/*
 * comm tracking
 */

struct perf_comm_event {
4037 4038
	struct task_struct	*task;
	char			*comm;
4039 4040 4041 4042 4043 4044 4045
	int			comm_size;

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
4046
	} event_id;
4047 4048
};

4049
static void perf_event_comm_output(struct perf_event *event,
4050 4051 4052
				     struct perf_comm_event *comm_event)
{
	struct perf_output_handle handle;
4053
	struct perf_sample_data sample;
4054
	int size = comm_event->event_id.header.size;
4055 4056 4057 4058
	int ret;

	perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
	ret = perf_output_begin(&handle, event,
4059
				comm_event->event_id.header.size);
4060 4061

	if (ret)
4062
		goto out;
4063

4064 4065
	comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
	comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
4066

4067
	perf_output_put(&handle, comm_event->event_id);
4068
	__output_copy(&handle, comm_event->comm,
4069
				   comm_event->comm_size);
4070 4071 4072

	perf_event__output_id_sample(event, &handle, &sample);

4073
	perf_output_end(&handle);
4074 4075
out:
	comm_event->event_id.header.size = size;
4076 4077
}

4078
static int perf_event_comm_match(struct perf_event *event)
4079
{
4080
	if (event->state < PERF_EVENT_STATE_INACTIVE)
4081 4082
		return 0;

4083
	if (!event_filter_match(event))
4084 4085
		return 0;

4086
	if (event->attr.comm)
4087 4088 4089 4090 4091
		return 1;

	return 0;
}

4092
static void perf_event_comm_ctx(struct perf_event_context *ctx,
4093 4094
				  struct perf_comm_event *comm_event)
{
4095
	struct perf_event *event;
4096

4097 4098 4099
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
		if (perf_event_comm_match(event))
			perf_event_comm_output(event, comm_event);
4100 4101 4102
	}
}

4103
static void perf_event_comm_event(struct perf_comm_event *comm_event)
4104 4105
{
	struct perf_cpu_context *cpuctx;
4106
	struct perf_event_context *ctx;
4107
	char comm[TASK_COMM_LEN];
4108
	unsigned int size;
4109
	struct pmu *pmu;
4110
	int ctxn;
4111

4112
	memset(comm, 0, sizeof(comm));
4113
	strlcpy(comm, comm_event->task->comm, sizeof(comm));
4114
	size = ALIGN(strlen(comm)+1, sizeof(u64));
4115 4116 4117 4118

	comm_event->comm = comm;
	comm_event->comm_size = size;

4119
	comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
4120
	rcu_read_lock();
4121
	list_for_each_entry_rcu(pmu, &pmus, entry) {
4122
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4123 4124
		if (cpuctx->active_pmu != pmu)
			goto next;
4125
		perf_event_comm_ctx(&cpuctx->ctx, comm_event);
4126 4127 4128

		ctxn = pmu->task_ctx_nr;
		if (ctxn < 0)
4129
			goto next;
4130 4131 4132 4133

		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		if (ctx)
			perf_event_comm_ctx(ctx, comm_event);
4134 4135
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
4136
	}
4137
	rcu_read_unlock();
4138 4139
}

4140
void perf_event_comm(struct task_struct *task)
4141
{
4142
	struct perf_comm_event comm_event;
4143 4144
	struct perf_event_context *ctx;
	int ctxn;
4145

4146 4147 4148 4149
	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (!ctx)
			continue;
4150

4151 4152
		perf_event_enable_on_exec(ctx);
	}
4153

4154
	if (!atomic_read(&nr_comm_events))
4155
		return;
4156

4157
	comm_event = (struct perf_comm_event){
4158
		.task	= task,
4159 4160
		/* .comm      */
		/* .comm_size */
4161
		.event_id  = {
4162
			.header = {
4163
				.type = PERF_RECORD_COMM,
4164 4165 4166 4167 4168
				.misc = 0,
				/* .size */
			},
			/* .pid */
			/* .tid */
4169 4170 4171
		},
	};

4172
	perf_event_comm_event(&comm_event);
4173 4174
}

4175 4176 4177 4178 4179
/*
 * mmap tracking
 */

struct perf_mmap_event {
4180 4181 4182 4183
	struct vm_area_struct	*vma;

	const char		*file_name;
	int			file_size;
4184 4185 4186 4187 4188 4189 4190 4191 4192

	struct {
		struct perf_event_header	header;

		u32				pid;
		u32				tid;
		u64				start;
		u64				len;
		u64				pgoff;
4193
	} event_id;
4194 4195
};

4196
static void perf_event_mmap_output(struct perf_event *event,
4197 4198 4199
				     struct perf_mmap_event *mmap_event)
{
	struct perf_output_handle handle;
4200
	struct perf_sample_data sample;
4201
	int size = mmap_event->event_id.header.size;
4202
	int ret;
4203

4204 4205
	perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
	ret = perf_output_begin(&handle, event,
4206
				mmap_event->event_id.header.size);
4207
	if (ret)
4208
		goto out;
4209

4210 4211
	mmap_event->event_id.pid = perf_event_pid(event, current);
	mmap_event->event_id.tid = perf_event_tid(event, current);
4212

4213
	perf_output_put(&handle, mmap_event->event_id);
4214
	__output_copy(&handle, mmap_event->file_name,
4215
				   mmap_event->file_size);
4216 4217 4218

	perf_event__output_id_sample(event, &handle, &sample);

4219
	perf_output_end(&handle);
4220 4221
out:
	mmap_event->event_id.header.size = size;
4222 4223
}

4224
static int perf_event_mmap_match(struct perf_event *event,
4225 4226
				   struct perf_mmap_event *mmap_event,
				   int executable)
4227
{
4228
	if (event->state < PERF_EVENT_STATE_INACTIVE)
4229 4230
		return 0;

4231
	if (!event_filter_match(event))
4232 4233
		return 0;

4234 4235
	if ((!executable && event->attr.mmap_data) ||
	    (executable && event->attr.mmap))
4236 4237 4238 4239 4240
		return 1;

	return 0;
}

4241
static void perf_event_mmap_ctx(struct perf_event_context *ctx,
4242 4243
				  struct perf_mmap_event *mmap_event,
				  int executable)
4244
{
4245
	struct perf_event *event;
4246

4247
	list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
4248
		if (perf_event_mmap_match(event, mmap_event, executable))
4249
			perf_event_mmap_output(event, mmap_event);
4250 4251 4252
	}
}

4253
static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
4254 4255
{
	struct perf_cpu_context *cpuctx;
4256
	struct perf_event_context *ctx;
4257 4258
	struct vm_area_struct *vma = mmap_event->vma;
	struct file *file = vma->vm_file;
4259 4260 4261
	unsigned int size;
	char tmp[16];
	char *buf = NULL;
4262
	const char *name;
4263
	struct pmu *pmu;
4264
	int ctxn;
4265

4266 4267
	memset(tmp, 0, sizeof(tmp));

4268
	if (file) {
4269
		/*
4270
		 * d_path works from the end of the rb backwards, so we
4271 4272 4273 4274
		 * need to add enough zero bytes after the string to handle
		 * the 64bit alignment we do later.
		 */
		buf = kzalloc(PATH_MAX + sizeof(u64), GFP_KERNEL);
4275 4276 4277 4278
		if (!buf) {
			name = strncpy(tmp, "//enomem", sizeof(tmp));
			goto got_name;
		}
4279
		name = d_path(&file->f_path, buf, PATH_MAX);
4280 4281 4282 4283 4284
		if (IS_ERR(name)) {
			name = strncpy(tmp, "//toolong", sizeof(tmp));
			goto got_name;
		}
	} else {
4285 4286 4287
		if (arch_vma_name(mmap_event->vma)) {
			name = strncpy(tmp, arch_vma_name(mmap_event->vma),
				       sizeof(tmp));
4288
			goto got_name;
4289
		}
4290 4291 4292 4293

		if (!vma->vm_mm) {
			name = strncpy(tmp, "[vdso]", sizeof(tmp));
			goto got_name;
4294 4295 4296 4297 4298 4299 4300 4301
		} else if (vma->vm_start <= vma->vm_mm->start_brk &&
				vma->vm_end >= vma->vm_mm->brk) {
			name = strncpy(tmp, "[heap]", sizeof(tmp));
			goto got_name;
		} else if (vma->vm_start <= vma->vm_mm->start_stack &&
				vma->vm_end >= vma->vm_mm->start_stack) {
			name = strncpy(tmp, "[stack]", sizeof(tmp));
			goto got_name;
4302 4303
		}

4304 4305 4306 4307 4308
		name = strncpy(tmp, "//anon", sizeof(tmp));
		goto got_name;
	}

got_name:
4309
	size = ALIGN(strlen(name)+1, sizeof(u64));
4310 4311 4312 4313

	mmap_event->file_name = name;
	mmap_event->file_size = size;

4314
	mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
4315

4316
	rcu_read_lock();
4317
	list_for_each_entry_rcu(pmu, &pmus, entry) {
4318
		cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
4319 4320
		if (cpuctx->active_pmu != pmu)
			goto next;
4321 4322
		perf_event_mmap_ctx(&cpuctx->ctx, mmap_event,
					vma->vm_flags & VM_EXEC);
4323 4324 4325

		ctxn = pmu->task_ctx_nr;
		if (ctxn < 0)
4326
			goto next;
4327 4328 4329 4330 4331 4332

		ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
		if (ctx) {
			perf_event_mmap_ctx(ctx, mmap_event,
					vma->vm_flags & VM_EXEC);
		}
4333 4334
next:
		put_cpu_ptr(pmu->pmu_cpu_context);
4335
	}
4336 4337
	rcu_read_unlock();

4338 4339 4340
	kfree(buf);
}

4341
void perf_event_mmap(struct vm_area_struct *vma)
4342
{
4343 4344
	struct perf_mmap_event mmap_event;

4345
	if (!atomic_read(&nr_mmap_events))
4346 4347 4348
		return;

	mmap_event = (struct perf_mmap_event){
4349
		.vma	= vma,
4350 4351
		/* .file_name */
		/* .file_size */
4352
		.event_id  = {
4353
			.header = {
4354
				.type = PERF_RECORD_MMAP,
4355
				.misc = PERF_RECORD_MISC_USER,
4356 4357 4358 4359
				/* .size */
			},
			/* .pid */
			/* .tid */
4360 4361
			.start  = vma->vm_start,
			.len    = vma->vm_end - vma->vm_start,
4362
			.pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
4363 4364 4365
		},
	};

4366
	perf_event_mmap_event(&mmap_event);
4367 4368
}

4369 4370 4371 4372
/*
 * IRQ throttle logging
 */

4373
static void perf_log_throttle(struct perf_event *event, int enable)
4374 4375
{
	struct perf_output_handle handle;
4376
	struct perf_sample_data sample;
4377 4378 4379 4380 4381
	int ret;

	struct {
		struct perf_event_header	header;
		u64				time;
4382
		u64				id;
4383
		u64				stream_id;
4384 4385
	} throttle_event = {
		.header = {
4386
			.type = PERF_RECORD_THROTTLE,
4387 4388 4389
			.misc = 0,
			.size = sizeof(throttle_event),
		},
4390
		.time		= perf_clock(),
4391 4392
		.id		= primary_event_id(event),
		.stream_id	= event->id,
4393 4394
	};

4395
	if (enable)
4396
		throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
4397

4398 4399 4400
	perf_event_header__init_id(&throttle_event.header, &sample, event);

	ret = perf_output_begin(&handle, event,
4401
				throttle_event.header.size);
4402 4403 4404 4405
	if (ret)
		return;

	perf_output_put(&handle, throttle_event);
4406
	perf_event__output_id_sample(event, &handle, &sample);
4407 4408 4409
	perf_output_end(&handle);
}

4410
/*
4411
 * Generic event overflow handling, sampling.
4412 4413
 */

4414
static int __perf_event_overflow(struct perf_event *event,
4415 4416
				   int throttle, struct perf_sample_data *data,
				   struct pt_regs *regs)
4417
{
4418 4419
	int events = atomic_read(&event->event_limit);
	struct hw_perf_event *hwc = &event->hw;
4420 4421
	int ret = 0;

4422 4423 4424 4425 4426 4427 4428
	/*
	 * Non-sampling counters might still use the PMI to fold short
	 * hardware counters, ignore those.
	 */
	if (unlikely(!is_sampling_event(event)))
		return 0;

4429 4430 4431 4432
	if (unlikely(hwc->interrupts >= max_samples_per_tick)) {
		if (throttle) {
			hwc->interrupts = MAX_INTERRUPTS;
			perf_log_throttle(event, 0);
4433 4434
			ret = 1;
		}
4435 4436
	} else
		hwc->interrupts++;
4437

4438
	if (event->attr.freq) {
4439
		u64 now = perf_clock();
4440
		s64 delta = now - hwc->freq_time_stamp;
4441

4442
		hwc->freq_time_stamp = now;
4443

4444 4445
		if (delta > 0 && delta < 2*TICK_NSEC)
			perf_adjust_period(event, delta, hwc->last_period);
4446 4447
	}

4448 4449
	/*
	 * XXX event_limit might not quite work as expected on inherited
4450
	 * events
4451 4452
	 */

4453 4454
	event->pending_kill = POLL_IN;
	if (events && atomic_dec_and_test(&event->event_limit)) {
4455
		ret = 1;
4456
		event->pending_kill = POLL_HUP;
4457 4458
		event->pending_disable = 1;
		irq_work_queue(&event->pending);
4459 4460
	}

4461
	if (event->overflow_handler)
4462
		event->overflow_handler(event, data, regs);
4463
	else
4464
		perf_event_output(event, data, regs);
4465

Peter Zijlstra's avatar
Peter Zijlstra committed
4466
	if (event->fasync && event->pending_kill) {
4467 4468
		event->pending_wakeup = 1;
		irq_work_queue(&event->pending);
Peter Zijlstra's avatar
Peter Zijlstra committed
4469 4470
	}

4471
	return ret;
4472 4473
}

4474
int perf_event_overflow(struct perf_event *event,
4475 4476
			  struct perf_sample_data *data,
			  struct pt_regs *regs)
4477
{
4478
	return __perf_event_overflow(event, 1, data, regs);
4479 4480
}

4481
/*
4482
 * Generic software event infrastructure
4483 4484
 */

4485 4486 4487 4488 4489 4490 4491 4492 4493 4494 4495
struct swevent_htable {
	struct swevent_hlist		*swevent_hlist;
	struct mutex			hlist_mutex;
	int				hlist_refcount;

	/* Recursion avoidance in each contexts */
	int				recursion[PERF_NR_CONTEXTS];
};

static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);

4496
/*
4497 4498
 * We directly increment event->count and keep a second value in
 * event->hw.period_left to count intervals. This period event
4499 4500 4501 4502
 * is kept in the range [-sample_period, 0] so that we can use the
 * sign as trigger.
 */

4503
static u64 perf_swevent_set_period(struct perf_event *event)
4504
{
4505
	struct hw_perf_event *hwc = &event->hw;
4506 4507 4508 4509 4510
	u64 period = hwc->last_period;
	u64 nr, offset;
	s64 old, val;

	hwc->last_period = hwc->sample_period;
4511 4512

again:
4513
	old = val = local64_read(&hwc->period_left);
4514 4515
	if (val < 0)
		return 0;
4516

4517 4518 4519
	nr = div64_u64(period + val, period);
	offset = nr * period;
	val -= offset;
4520
	if (local64_cmpxchg(&hwc->period_left, old, val) != old)
4521
		goto again;
4522

4523
	return nr;
4524 4525
}

4526
static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
4527
				    struct perf_sample_data *data,
4528
				    struct pt_regs *regs)
4529
{
4530
	struct hw_perf_event *hwc = &event->hw;
4531
	int throttle = 0;
4532

4533 4534
	if (!overflow)
		overflow = perf_swevent_set_period(event);
4535

4536 4537
	if (hwc->interrupts == MAX_INTERRUPTS)
		return;
4538

4539
	for (; overflow; overflow--) {
4540
		if (__perf_event_overflow(event, throttle,
4541
					    data, regs)) {
4542 4543 4544 4545 4546 4547
			/*
			 * We inhibit the overflow from happening when
			 * hwc->interrupts == MAX_INTERRUPTS.
			 */
			break;
		}
4548
		throttle = 1;
4549
	}
4550 4551
}

4552
static void perf_swevent_event(struct perf_event *event, u64 nr,
4553
			       struct perf_sample_data *data,
4554
			       struct pt_regs *regs)
4555
{
4556
	struct hw_perf_event *hwc = &event->hw;
4557

4558
	local64_add(nr, &event->count);
4559

4560 4561 4562
	if (!regs)
		return;

4563
	if (!is_sampling_event(event))
4564
		return;
4565

4566 4567 4568 4569 4570 4571
	if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
		data->period = nr;
		return perf_swevent_overflow(event, 1, data, regs);
	} else
		data->period = event->hw.last_period;

4572
	if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
4573
		return perf_swevent_overflow(event, 1, data, regs);
4574

4575
	if (local64_add_negative(nr, &hwc->period_left))
4576
		return;
4577

4578
	perf_swevent_overflow(event, 0, data, regs);
4579 4580
}

4581 4582 4583
static int perf_exclude_event(struct perf_event *event,
			      struct pt_regs *regs)
{
4584
	if (event->hw.state & PERF_HES_STOPPED)
4585
		return 1;
4586

4587 4588 4589 4590 4591 4592 4593 4594 4595 4596 4597
	if (regs) {
		if (event->attr.exclude_user && user_mode(regs))
			return 1;

		if (event->attr.exclude_kernel && !user_mode(regs))
			return 1;
	}

	return 0;
}

4598
static int perf_swevent_match(struct perf_event *event,
Peter Zijlstra's avatar
Peter Zijlstra committed
4599
				enum perf_type_id type,
4600 4601 4602
				u32 event_id,
				struct perf_sample_data *data,
				struct pt_regs *regs)
4603
{
4604
	if (event->attr.type != type)
4605
		return 0;
4606

4607
	if (event->attr.config != event_id)
4608 4609
		return 0;

4610 4611
	if (perf_exclude_event(event, regs))
		return 0;
4612 4613 4614 4615

	return 1;
}

4616 4617 4618 4619 4620 4621 4622
static inline u64 swevent_hash(u64 type, u32 event_id)
{
	u64 val = event_id | (type << 32);

	return hash_64(val, SWEVENT_HLIST_BITS);
}

4623 4624
static inline struct hlist_head *
__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
4625
{
4626 4627 4628 4629
	u64 hash = swevent_hash(type, event_id);

	return &hlist->heads[hash];
}
4630

4631 4632
/* For the read side: events when they trigger */
static inline struct hlist_head *
4633
find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
4634 4635
{
	struct swevent_hlist *hlist;
4636

4637
	hlist = rcu_dereference(swhash->swevent_hlist);
4638 4639 4640
	if (!hlist)
		return NULL;

4641 4642 4643 4644 4645
	return __find_swevent_head(hlist, type, event_id);
}

/* For the event head insertion and removal in the hlist */
static inline struct hlist_head *
4646
find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
4647 4648 4649 4650 4651 4652 4653 4654 4655 4656
{
	struct swevent_hlist *hlist;
	u32 event_id = event->attr.config;
	u64 type = event->attr.type;

	/*
	 * Event scheduling is always serialized against hlist allocation
	 * and release. Which makes the protected version suitable here.
	 * The context lock guarantees that.
	 */
4657
	hlist = rcu_dereference_protected(swhash->swevent_hlist,
4658 4659 4660 4661 4662
					  lockdep_is_held(&event->ctx->lock));
	if (!hlist)
		return NULL;

	return __find_swevent_head(hlist, type, event_id);
4663 4664 4665
}

static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
4666
				    u64 nr,
4667 4668
				    struct perf_sample_data *data,
				    struct pt_regs *regs)
4669
{
4670
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4671
	struct perf_event *event;
4672 4673
	struct hlist_node *node;
	struct hlist_head *head;
4674

4675
	rcu_read_lock();
4676
	head = find_swevent_head_rcu(swhash, type, event_id);
4677 4678 4679 4680
	if (!head)
		goto end;

	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
4681
		if (perf_swevent_match(event, type, event_id, data, regs))
4682
			perf_swevent_event(event, nr, data, regs);
4683
	}
4684 4685
end:
	rcu_read_unlock();
4686 4687
}

4688
int perf_swevent_get_recursion_context(void)
4689
{
4690
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4691

4692
	return get_recursion_context(swhash->recursion);
4693
}
Ingo Molnar's avatar
Ingo Molnar committed
4694
EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
4695

4696
inline void perf_swevent_put_recursion_context(int rctx)
4697
{
4698
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4699

4700
	put_recursion_context(swhash->recursion, rctx);
4701
}
4702

4703
void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
4704
{
4705
	struct perf_sample_data data;
4706 4707
	int rctx;

4708
	preempt_disable_notrace();
4709 4710 4711
	rctx = perf_swevent_get_recursion_context();
	if (rctx < 0)
		return;
4712

4713
	perf_sample_data_init(&data, addr);
4714

4715
	do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
4716 4717

	perf_swevent_put_recursion_context(rctx);
4718
	preempt_enable_notrace();
4719 4720
}

4721
static void perf_swevent_read(struct perf_event *event)
4722 4723 4724
{
}

4725
static int perf_swevent_add(struct perf_event *event, int flags)
4726
{
4727
	struct swevent_htable *swhash = &__get_cpu_var(swevent_htable);
4728
	struct hw_perf_event *hwc = &event->hw;
4729 4730
	struct hlist_head *head;

4731
	if (is_sampling_event(event)) {
4732
		hwc->last_period = hwc->sample_period;
4733
		perf_swevent_set_period(event);
4734
	}
4735

4736 4737
	hwc->state = !(flags & PERF_EF_START);

4738
	head = find_swevent_head(swhash, event);
4739 4740 4741 4742 4743
	if (WARN_ON_ONCE(!head))
		return -EINVAL;

	hlist_add_head_rcu(&event->hlist_entry, head);

4744 4745 4746
	return 0;
}

4747
static void perf_swevent_del(struct perf_event *event, int flags)
4748
{
4749
	hlist_del_rcu(&event->hlist_entry);
4750 4751
}

4752
static void perf_swevent_start(struct perf_event *event, int flags)
4753
{
4754
	event->hw.state = 0;
4755
}
4756

4757
static void perf_swevent_stop(struct perf_event *event, int flags)
4758
{
4759
	event->hw.state = PERF_HES_STOPPED;
4760 4761
}

4762 4763
/* Deref the hlist from the update side */
static inline struct swevent_hlist *
4764
swevent_hlist_deref(struct swevent_htable *swhash)
4765
{
4766 4767
	return rcu_dereference_protected(swhash->swevent_hlist,
					 lockdep_is_held(&swhash->hlist_mutex));
4768 4769
}

4770
static void swevent_hlist_release(struct swevent_htable *swhash)
4771
{
4772
	struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
4773

4774
	if (!hlist)
4775 4776
		return;

4777
	rcu_assign_pointer(swhash->swevent_hlist, NULL);
4778
	kfree_rcu(hlist, rcu_head);
4779 4780 4781 4782
}

static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
{
4783
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
4784

4785
	mutex_lock(&swhash->hlist_mutex);
4786

4787 4788
	if (!--swhash->hlist_refcount)
		swevent_hlist_release(swhash);
4789

4790
	mutex_unlock(&swhash->hlist_mutex);
4791 4792 4793 4794 4795 4796 4797 4798 4799 4800 4801 4802 4803 4804 4805 4806 4807
}

static void swevent_hlist_put(struct perf_event *event)
{
	int cpu;

	if (event->cpu != -1) {
		swevent_hlist_put_cpu(event, event->cpu);
		return;
	}

	for_each_possible_cpu(cpu)
		swevent_hlist_put_cpu(event, cpu);
}

static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
{
4808
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
4809 4810
	int err = 0;

4811
	mutex_lock(&swhash->hlist_mutex);
4812

4813
	if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
4814 4815 4816 4817 4818 4819 4820
		struct swevent_hlist *hlist;

		hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
		if (!hlist) {
			err = -ENOMEM;
			goto exit;
		}
4821
		rcu_assign_pointer(swhash->swevent_hlist, hlist);
4822
	}
4823
	swhash->hlist_refcount++;
Peter Zijlstra's avatar
Peter Zijlstra committed
4824
exit:
4825
	mutex_unlock(&swhash->hlist_mutex);
4826 4827 4828 4829 4830 4831 4832 4833 4834 4835 4836 4837 4838 4839 4840 4841 4842 4843 4844 4845 4846 4847 4848

	return err;
}

static int swevent_hlist_get(struct perf_event *event)
{
	int err;
	int cpu, failed_cpu;

	if (event->cpu != -1)
		return swevent_hlist_get_cpu(event, event->cpu);

	get_online_cpus();
	for_each_possible_cpu(cpu) {
		err = swevent_hlist_get_cpu(event, cpu);
		if (err) {
			failed_cpu = cpu;
			goto fail;
		}
	}
	put_online_cpus();

	return 0;
Peter Zijlstra's avatar
Peter Zijlstra committed
4849
fail:
4850 4851 4852 4853 4854 4855 4856 4857 4858 4859
	for_each_possible_cpu(cpu) {
		if (cpu == failed_cpu)
			break;
		swevent_hlist_put_cpu(event, cpu);
	}

	put_online_cpus();
	return err;
}

4860
struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
4861

4862 4863 4864
static void sw_perf_event_destroy(struct perf_event *event)
{
	u64 event_id = event->attr.config;
4865

4866 4867
	WARN_ON(event->parent);

Peter Zijlstra's avatar
Peter Zijlstra committed
4868
	jump_label_dec(&perf_swevent_enabled[event_id]);
4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882 4883 4884 4885 4886 4887
	swevent_hlist_put(event);
}

static int perf_swevent_init(struct perf_event *event)
{
	int event_id = event->attr.config;

	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

	switch (event_id) {
	case PERF_COUNT_SW_CPU_CLOCK:
	case PERF_COUNT_SW_TASK_CLOCK:
		return -ENOENT;

	default:
		break;
	}

4888
	if (event_id >= PERF_COUNT_SW_MAX)
4889 4890 4891 4892 4893 4894 4895 4896 4897
		return -ENOENT;

	if (!event->parent) {
		int err;

		err = swevent_hlist_get(event);
		if (err)
			return err;

Peter Zijlstra's avatar
Peter Zijlstra committed
4898
		jump_label_inc(&perf_swevent_enabled[event_id]);
4899 4900 4901 4902 4903 4904 4905
		event->destroy = sw_perf_event_destroy;
	}

	return 0;
}

static struct pmu perf_swevent = {
4906
	.task_ctx_nr	= perf_sw_context,
4907

4908
	.event_init	= perf_swevent_init,
4909 4910 4911 4912
	.add		= perf_swevent_add,
	.del		= perf_swevent_del,
	.start		= perf_swevent_start,
	.stop		= perf_swevent_stop,
4913 4914 4915
	.read		= perf_swevent_read,
};

4916 4917
#ifdef CONFIG_EVENT_TRACING

4918 4919 4920 4921 4922 4923 4924 4925 4926 4927 4928 4929 4930 4931
static int perf_tp_filter_match(struct perf_event *event,
				struct perf_sample_data *data)
{
	void *record = data->raw->data;

	if (likely(!event->filter) || filter_match_preds(event->filter, record))
		return 1;
	return 0;
}

static int perf_tp_event_match(struct perf_event *event,
				struct perf_sample_data *data,
				struct pt_regs *regs)
{
4932 4933
	if (event->hw.state & PERF_HES_STOPPED)
		return 0;
4934 4935 4936 4937
	/*
	 * All tracepoints are from kernel-space.
	 */
	if (event->attr.exclude_kernel)
4938 4939 4940 4941 4942 4943 4944 4945 4946
		return 0;

	if (!perf_tp_filter_match(event, data))
		return 0;

	return 1;
}

void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
4947
		   struct pt_regs *regs, struct hlist_head *head, int rctx)
4948 4949
{
	struct perf_sample_data data;
4950 4951 4952
	struct perf_event *event;
	struct hlist_node *node;

4953 4954 4955 4956 4957 4958 4959 4960
	struct perf_raw_record raw = {
		.size = entry_size,
		.data = record,
	};

	perf_sample_data_init(&data, addr);
	data.raw = &raw;

4961 4962
	hlist_for_each_entry_rcu(event, node, head, hlist_entry) {
		if (perf_tp_event_match(event, &data, regs))
4963
			perf_swevent_event(event, count, &data, regs);
4964
	}
4965 4966

	perf_swevent_put_recursion_context(rctx);
4967 4968 4969
}
EXPORT_SYMBOL_GPL(perf_tp_event);

4970
static void tp_perf_event_destroy(struct perf_event *event)
4971
{
4972
	perf_trace_destroy(event);
4973 4974
}

4975
static int perf_tp_event_init(struct perf_event *event)
4976
{
4977 4978
	int err;

4979 4980 4981
	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -ENOENT;

4982 4983
	err = perf_trace_init(event);
	if (err)
4984
		return err;
4985

4986
	event->destroy = tp_perf_event_destroy;
4987

4988 4989 4990 4991
	return 0;
}

static struct pmu perf_tracepoint = {
4992 4993
	.task_ctx_nr	= perf_sw_context,

4994
	.event_init	= perf_tp_event_init,
4995 4996 4997 4998
	.add		= perf_trace_add,
	.del		= perf_trace_del,
	.start		= perf_swevent_start,
	.stop		= perf_swevent_stop,
4999 5000 5001 5002 5003
	.read		= perf_swevent_read,
};

static inline void perf_tp_register(void)
{
Peter Zijlstra's avatar
Peter Zijlstra committed
5004
	perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
5005
}
5006 5007 5008 5009 5010 5011 5012 5013 5014 5015 5016 5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	char *filter_str;
	int ret;

	if (event->attr.type != PERF_TYPE_TRACEPOINT)
		return -EINVAL;

	filter_str = strndup_user(arg, PAGE_SIZE);
	if (IS_ERR(filter_str))
		return PTR_ERR(filter_str);

	ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);

	kfree(filter_str);
	return ret;
}

static void perf_event_free_filter(struct perf_event *event)
{
	ftrace_profile_free_filter(event);
}

5030
#else
5031

5032
static inline void perf_tp_register(void)
5033 5034
{
}
5035 5036 5037 5038 5039 5040 5041 5042 5043 5044

static int perf_event_set_filter(struct perf_event *event, void __user *arg)
{
	return -ENOENT;
}

static void perf_event_free_filter(struct perf_event *event)
{
}

5045
#endif /* CONFIG_EVENT_TRACING */
5046

5047
#ifdef CONFIG_HAVE_HW_BREAKPOINT
5048
void perf_bp_event(struct perf_event *bp, void *data)
5049
{
5050 5051 5052
	struct perf_sample_data sample;
	struct pt_regs *regs = data;

5053
	perf_sample_data_init(&sample, bp->attr.bp_addr);
5054

5055
	if (!bp->hw.state && !perf_exclude_event(bp, regs))
5056
		perf_swevent_event(bp, 1, &sample, regs);
5057 5058 5059
}
#endif

5060 5061 5062
/*
 * hrtimer based swevent callback
 */
5063

5064
static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
5065
{
5066 5067 5068 5069 5070
	enum hrtimer_restart ret = HRTIMER_RESTART;
	struct perf_sample_data data;
	struct pt_regs *regs;
	struct perf_event *event;
	u64 period;
5071

5072
	event = container_of(hrtimer, struct perf_event, hw.hrtimer);
5073 5074 5075 5076

	if (event->state != PERF_EVENT_STATE_ACTIVE)
		return HRTIMER_NORESTART;

5077
	event->pmu->read(event);
5078

5079 5080 5081 5082 5083 5084
	perf_sample_data_init(&data, 0);
	data.period = event->hw.last_period;
	regs = get_irq_regs();

	if (regs && !perf_exclude_event(event, regs)) {
		if (!(event->attr.exclude_idle && current->pid == 0))
5085
			if (perf_event_overflow(event, &data, regs))
5086 5087
				ret = HRTIMER_NORESTART;
	}
5088

5089 5090
	period = max_t(u64, 10000, event->hw.sample_period);
	hrtimer_forward_now(hrtimer, ns_to_ktime(period));
5091

5092
	return ret;
5093 5094
}

5095
static void perf_swevent_start_hrtimer(struct perf_event *event)
5096
{
5097
	struct hw_perf_event *hwc = &event->hw;
5098 5099 5100 5101
	s64 period;

	if (!is_sampling_event(event))
		return;
5102

5103 5104 5105 5106
	period = local64_read(&hwc->period_left);
	if (period) {
		if (period < 0)
			period = 10000;
Peter Zijlstra's avatar
Peter Zijlstra committed
5107

5108 5109 5110 5111 5112
		local64_set(&hwc->period_left, 0);
	} else {
		period = max_t(u64, 10000, hwc->sample_period);
	}
	__hrtimer_start_range_ns(&hwc->hrtimer,
5113
				ns_to_ktime(period), 0,
5114
				HRTIMER_MODE_REL_PINNED, 0);
5115
}
5116 5117

static void perf_swevent_cancel_hrtimer(struct perf_event *event)
5118
{
5119 5120
	struct hw_perf_event *hwc = &event->hw;

5121
	if (is_sampling_event(event)) {
5122
		ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
Peter Zijlstra's avatar
Peter Zijlstra committed
5123
		local64_set(&hwc->period_left, ktime_to_ns(remaining));
5124 5125 5126

		hrtimer_cancel(&hwc->hrtimer);
	}
5127 5128
}

5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145 5146 5147 5148 5149 5150 5151 5152
static void perf_swevent_init_hrtimer(struct perf_event *event)
{
	struct hw_perf_event *hwc = &event->hw;

	if (!is_sampling_event(event))
		return;

	hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
	hwc->hrtimer.function = perf_swevent_hrtimer;

	/*
	 * Since hrtimers have a fixed rate, we can do a static freq->period
	 * mapping and avoid the whole period adjust feedback stuff.
	 */
	if (event->attr.freq) {
		long freq = event->attr.sample_freq;

		event->attr.sample_period = NSEC_PER_SEC / freq;
		hwc->sample_period = event->attr.sample_period;
		local64_set(&hwc->period_left, hwc->sample_period);
		event->attr.freq = 0;
	}
}

5153 5154 5155 5156 5157
/*
 * Software event: cpu wall time clock
 */

static void cpu_clock_event_update(struct perf_event *event)
5158
{
5159 5160 5161
	s64 prev;
	u64 now;

5162
	now = local_clock();
5163 5164
	prev = local64_xchg(&event->hw.prev_count, now);
	local64_add(now - prev, &event->count);
5165 5166
}

5167
static void cpu_clock_event_start(struct perf_event *event, int flags)
5168
{
5169
	local64_set(&event->hw.prev_count, local_clock());
5170 5171 5172
	perf_swevent_start_hrtimer(event);
}

5173
static void cpu_clock_event_stop(struct perf_event *event, int flags)
5174
{
5175 5176 5177
	perf_swevent_cancel_hrtimer(event);
	cpu_clock_event_update(event);
}
5178

5179 5180 5181 5182 5183 5184 5185 5186 5187 5188 5189 5190 5191
static int cpu_clock_event_add(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_START)
		cpu_clock_event_start(event, flags);

	return 0;
}

static void cpu_clock_event_del(struct perf_event *event, int flags)
{
	cpu_clock_event_stop(event, flags);
}

5192 5193 5194 5195
static void cpu_clock_event_read(struct perf_event *event)
{
	cpu_clock_event_update(event);
}
5196

5197 5198 5199 5200 5201 5202 5203 5204
static int cpu_clock_event_init(struct perf_event *event)
{
	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

	if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
		return -ENOENT;

5205 5206
	perf_swevent_init_hrtimer(event);

5207
	return 0;
5208 5209
}

5210
static struct pmu perf_cpu_clock = {
5211 5212
	.task_ctx_nr	= perf_sw_context,

5213
	.event_init	= cpu_clock_event_init,
5214 5215 5216 5217
	.add		= cpu_clock_event_add,
	.del		= cpu_clock_event_del,
	.start		= cpu_clock_event_start,
	.stop		= cpu_clock_event_stop,
5218 5219 5220 5221 5222 5223 5224 5225
	.read		= cpu_clock_event_read,
};

/*
 * Software event: task time clock
 */

static void task_clock_event_update(struct perf_event *event, u64 now)
5226
{
5227 5228
	u64 prev;
	s64 delta;
5229

5230 5231 5232 5233
	prev = local64_xchg(&event->hw.prev_count, now);
	delta = now - prev;
	local64_add(delta, &event->count);
}
5234

5235
static void task_clock_event_start(struct perf_event *event, int flags)
5236
{
5237
	local64_set(&event->hw.prev_count, event->ctx->time);
5238 5239 5240
	perf_swevent_start_hrtimer(event);
}

5241
static void task_clock_event_stop(struct perf_event *event, int flags)
5242 5243 5244
{
	perf_swevent_cancel_hrtimer(event);
	task_clock_event_update(event, event->ctx->time);
5245 5246 5247 5248 5249 5250
}

static int task_clock_event_add(struct perf_event *event, int flags)
{
	if (flags & PERF_EF_START)
		task_clock_event_start(event, flags);
5251

5252 5253 5254 5255 5256 5257
	return 0;
}

static void task_clock_event_del(struct perf_event *event, int flags)
{
	task_clock_event_stop(event, PERF_EF_UPDATE);
5258 5259 5260 5261
}

static void task_clock_event_read(struct perf_event *event)
{
5262 5263 5264
	u64 now = perf_clock();
	u64 delta = now - event->ctx->timestamp;
	u64 time = event->ctx->time + delta;
5265 5266 5267 5268 5269

	task_clock_event_update(event, time);
}

static int task_clock_event_init(struct perf_event *event)
5270
{
5271 5272 5273 5274 5275 5276
	if (event->attr.type != PERF_TYPE_SOFTWARE)
		return -ENOENT;

	if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
		return -ENOENT;

5277 5278
	perf_swevent_init_hrtimer(event);

5279
	return 0;
5280 5281
}

5282
static struct pmu perf_task_clock = {
5283 5284
	.task_ctx_nr	= perf_sw_context,

5285
	.event_init	= task_clock_event_init,
5286 5287 5288 5289
	.add		= task_clock_event_add,
	.del		= task_clock_event_del,
	.start		= task_clock_event_start,
	.stop		= task_clock_event_stop,
5290 5291
	.read		= task_clock_event_read,
};
5292

Peter Zijlstra's avatar
Peter Zijlstra committed
5293
static void perf_pmu_nop_void(struct pmu *pmu)
5294 5295
{
}
5296

Peter Zijlstra's avatar
Peter Zijlstra committed
5297
static int perf_pmu_nop_int(struct pmu *pmu)
5298
{
Peter Zijlstra's avatar
Peter Zijlstra committed
5299
	return 0;
5300 5301
}

Peter Zijlstra's avatar
Peter Zijlstra committed
5302
static void perf_pmu_start_txn(struct pmu *pmu)
5303
{
Peter Zijlstra's avatar
Peter Zijlstra committed
5304
	perf_pmu_disable(pmu);
5305 5306
}

Peter Zijlstra's avatar
Peter Zijlstra committed
5307 5308 5309 5310 5311
static int perf_pmu_commit_txn(struct pmu *pmu)
{
	perf_pmu_enable(pmu);
	return 0;
}
5312

Peter Zijlstra's avatar
Peter Zijlstra committed
5313
static void perf_pmu_cancel_txn(struct pmu *pmu)
5314
{
Peter Zijlstra's avatar
Peter Zijlstra committed
5315
	perf_pmu_enable(pmu);
5316 5317
}

5318 5319 5320 5321 5322
/*
 * Ensures all contexts with the same task_ctx_nr have the same
 * pmu_cpu_context too.
 */
static void *find_pmu_context(int ctxn)
5323
{
5324
	struct pmu *pmu;
5325

5326 5327
	if (ctxn < 0)
		return NULL;
5328

5329 5330 5331 5332
	list_for_each_entry(pmu, &pmus, entry) {
		if (pmu->task_ctx_nr == ctxn)
			return pmu->pmu_cpu_context;
	}
5333

5334
	return NULL;
5335 5336
}

5337
static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
5338
{
5339 5340 5341 5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352 5353
	int cpu;

	for_each_possible_cpu(cpu) {
		struct perf_cpu_context *cpuctx;

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);

		if (cpuctx->active_pmu == old_pmu)
			cpuctx->active_pmu = pmu;
	}
}

static void free_pmu_context(struct pmu *pmu)
{
	struct pmu *i;
5354

5355
	mutex_lock(&pmus_lock);
5356
	/*
5357
	 * Like a real lame refcount.
5358
	 */
5359 5360 5361
	list_for_each_entry(i, &pmus, entry) {
		if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
			update_pmu_context(i, pmu);
5362
			goto out;
5363
		}
5364
	}
5365

5366
	free_percpu(pmu->pmu_cpu_context);
5367 5368
out:
	mutex_unlock(&pmus_lock);
5369
}
Peter Zijlstra's avatar
Peter Zijlstra committed
5370
static struct idr pmu_idr;
5371

Peter Zijlstra's avatar
Peter Zijlstra committed
5372 5373 5374 5375 5376 5377 5378 5379 5380 5381 5382 5383 5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410 5411 5412 5413 5414 5415 5416 5417 5418 5419 5420 5421 5422 5423
static ssize_t
type_show(struct device *dev, struct device_attribute *attr, char *page)
{
	struct pmu *pmu = dev_get_drvdata(dev);

	return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
}

static struct device_attribute pmu_dev_attrs[] = {
       __ATTR_RO(type),
       __ATTR_NULL,
};

static int pmu_bus_running;
static struct bus_type pmu_bus = {
	.name		= "event_source",
	.dev_attrs	= pmu_dev_attrs,
};

static void pmu_dev_release(struct device *dev)
{
	kfree(dev);
}

static int pmu_dev_alloc(struct pmu *pmu)
{
	int ret = -ENOMEM;

	pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
	if (!pmu->dev)
		goto out;

	device_initialize(pmu->dev);
	ret = dev_set_name(pmu->dev, "%s", pmu->name);
	if (ret)
		goto free_dev;

	dev_set_drvdata(pmu->dev, pmu);
	pmu->dev->bus = &pmu_bus;
	pmu->dev->release = pmu_dev_release;
	ret = device_add(pmu->dev);
	if (ret)
		goto free_dev;

out:
	return ret;

free_dev:
	put_device(pmu->dev);
	goto out;
}

5424
static struct lock_class_key cpuctx_mutex;
5425
static struct lock_class_key cpuctx_lock;
5426

Peter Zijlstra's avatar
Peter Zijlstra committed
5427
int perf_pmu_register(struct pmu *pmu, char *name, int type)
5428
{
5429
	int cpu, ret;
5430

5431
	mutex_lock(&pmus_lock);
Peter Zijlstra's avatar
Peter Zijlstra committed
5432 5433 5434 5435
	ret = -ENOMEM;
	pmu->pmu_disable_count = alloc_percpu(int);
	if (!pmu->pmu_disable_count)
		goto unlock;
5436

Peter Zijlstra's avatar
Peter Zijlstra committed
5437 5438 5439 5440 5441 5442 5443 5444 5445 5446 5447 5448 5449 5450 5451 5452 5453 5454
	pmu->type = -1;
	if (!name)
		goto skip_type;
	pmu->name = name;

	if (type < 0) {
		int err = idr_pre_get(&pmu_idr, GFP_KERNEL);
		if (!err)
			goto free_pdc;

		err = idr_get_new_above(&pmu_idr, pmu, PERF_TYPE_MAX, &type);
		if (err) {
			ret = err;
			goto free_pdc;
		}
	}
	pmu->type = type;

Peter Zijlstra's avatar
Peter Zijlstra committed
5455 5456 5457 5458 5459 5460
	if (pmu_bus_running) {
		ret = pmu_dev_alloc(pmu);
		if (ret)
			goto free_idr;
	}

Peter Zijlstra's avatar
Peter Zijlstra committed
5461
skip_type:
5462 5463 5464
	pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
	if (pmu->pmu_cpu_context)
		goto got_cpu_context;
5465

5466 5467
	pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
	if (!pmu->pmu_cpu_context)
Peter Zijlstra's avatar
Peter Zijlstra committed
5468
		goto free_dev;
5469

5470 5471 5472 5473
	for_each_possible_cpu(cpu) {
		struct perf_cpu_context *cpuctx;

		cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
5474
		__perf_event_init_context(&cpuctx->ctx);
5475
		lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
5476
		lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
5477
		cpuctx->ctx.type = cpu_context;
5478
		cpuctx->ctx.pmu = pmu;
5479 5480
		cpuctx->jiffies_interval = 1;
		INIT_LIST_HEAD(&cpuctx->rotation_list);
5481
		cpuctx->active_pmu = pmu;
5482
	}
5483

5484
got_cpu_context:
Peter Zijlstra's avatar
Peter Zijlstra committed
5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497 5498
	if (!pmu->start_txn) {
		if (pmu->pmu_enable) {
			/*
			 * If we have pmu_enable/pmu_disable calls, install
			 * transaction stubs that use that to try and batch
			 * hardware accesses.
			 */
			pmu->start_txn  = perf_pmu_start_txn;
			pmu->commit_txn = perf_pmu_commit_txn;
			pmu->cancel_txn = perf_pmu_cancel_txn;
		} else {
			pmu->start_txn  = perf_pmu_nop_void;
			pmu->commit_txn = perf_pmu_nop_int;
			pmu->cancel_txn = perf_pmu_nop_void;
5499
		}
5500
	}
5501

Peter Zijlstra's avatar
Peter Zijlstra committed
5502 5503 5504 5505 5506
	if (!pmu->pmu_enable) {
		pmu->pmu_enable  = perf_pmu_nop_void;
		pmu->pmu_disable = perf_pmu_nop_void;
	}

5507
	list_add_rcu(&pmu->entry, &pmus);
Peter Zijlstra's avatar
Peter Zijlstra committed
5508 5509
	ret = 0;
unlock:
5510 5511
	mutex_unlock(&pmus_lock);

Peter Zijlstra's avatar
Peter Zijlstra committed
5512
	return ret;
5513

Peter Zijlstra's avatar
Peter Zijlstra committed
5514 5515 5516 5517
free_dev:
	device_del(pmu->dev);
	put_device(pmu->dev);

Peter Zijlstra's avatar
Peter Zijlstra committed
5518 5519 5520 5521
free_idr:
	if (pmu->type >= PERF_TYPE_MAX)
		idr_remove(&pmu_idr, pmu->type);

5522 5523 5524
free_pdc:
	free_percpu(pmu->pmu_disable_count);
	goto unlock;
5525 5526
}

5527
void perf_pmu_unregister(struct pmu *pmu)
5528
{
5529 5530 5531
	mutex_lock(&pmus_lock);
	list_del_rcu(&pmu->entry);
	mutex_unlock(&pmus_lock);
5532

5533
	/*
5534 5535
	 * We dereference the pmu list under both SRCU and regular RCU, so
	 * synchronize against both of those.
5536
	 */
5537
	synchronize_srcu(&pmus_srcu);
5538
	synchronize_rcu();
5539

Peter Zijlstra's avatar
Peter Zijlstra committed
5540
	free_percpu(pmu->pmu_disable_count);
Peter Zijlstra's avatar
Peter Zijlstra committed
5541 5542
	if (pmu->type >= PERF_TYPE_MAX)
		idr_remove(&pmu_idr, pmu->type);
Peter Zijlstra's avatar
Peter Zijlstra committed
5543 5544
	device_del(pmu->dev);
	put_device(pmu->dev);
5545
	free_pmu_context(pmu);
5546
}
5547

5548 5549 5550 5551
struct pmu *perf_init_event(struct perf_event *event)
{
	struct pmu *pmu = NULL;
	int idx;
5552
	int ret;
5553 5554

	idx = srcu_read_lock(&pmus_srcu);
Peter Zijlstra's avatar
Peter Zijlstra committed
5555 5556 5557 5558

	rcu_read_lock();
	pmu = idr_find(&pmu_idr, event->attr.type);
	rcu_read_unlock();
5559
	if (pmu) {
5560
		event->pmu = pmu;
5561 5562 5563
		ret = pmu->event_init(event);
		if (ret)
			pmu = ERR_PTR(ret);
Peter Zijlstra's avatar
Peter Zijlstra committed
5564
		goto unlock;
5565
	}
Peter Zijlstra's avatar
Peter Zijlstra committed
5566

5567
	list_for_each_entry_rcu(pmu, &pmus, entry) {
5568
		event->pmu = pmu;
5569
		ret = pmu->event_init(event);
5570
		if (!ret)
Peter Zijlstra's avatar
Peter Zijlstra committed
5571
			goto unlock;
5572

5573 5574
		if (ret != -ENOENT) {
			pmu = ERR_PTR(ret);
Peter Zijlstra's avatar
Peter Zijlstra committed
5575
			goto unlock;
5576
		}
5577
	}
Peter Zijlstra's avatar
Peter Zijlstra committed
5578 5579
	pmu = ERR_PTR(-ENOENT);
unlock:
5580
	srcu_read_unlock(&pmus_srcu, idx);
5581

5582
	return pmu;
5583 5584
}

5585
/*
5586
 * Allocate and initialize a event structure
5587
 */
5588
static struct perf_event *
5589
perf_event_alloc(struct perf_event_attr *attr, int cpu,
5590 5591 5592
		 struct task_struct *task,
		 struct perf_event *group_leader,
		 struct perf_event *parent_event,
5593 5594
		 perf_overflow_handler_t overflow_handler,
		 void *context)
5595
{
Peter Zijlstra's avatar
Peter Zijlstra committed
5596
	struct pmu *pmu;
5597 5598
	struct perf_event *event;
	struct hw_perf_event *hwc;
5599
	long err;
5600

5601 5602 5603 5604 5605
	if ((unsigned)cpu >= nr_cpu_ids) {
		if (!task || cpu != -1)
			return ERR_PTR(-EINVAL);
	}

5606
	event = kzalloc(sizeof(*event), GFP_KERNEL);
5607
	if (!event)
5608
		return ERR_PTR(-ENOMEM);
5609

5610
	/*
5611
	 * Single events are their own group leaders, with an
5612 5613 5614
	 * empty sibling list:
	 */
	if (!group_leader)
5615
		group_leader = event;
5616

5617 5618
	mutex_init(&event->child_mutex);
	INIT_LIST_HEAD(&event->child_list);
5619

5620 5621 5622 5623
	INIT_LIST_HEAD(&event->group_entry);
	INIT_LIST_HEAD(&event->event_entry);
	INIT_LIST_HEAD(&event->sibling_list);
	init_waitqueue_head(&event->waitq);
5624
	init_irq_work(&event->pending, perf_pending_event);
5625

5626
	mutex_init(&event->mmap_mutex);
5627

5628 5629 5630 5631 5632
	event->cpu		= cpu;
	event->attr		= *attr;
	event->group_leader	= group_leader;
	event->pmu		= NULL;
	event->oncpu		= -1;
5633

5634
	event->parent		= parent_event;
5635

5636 5637
	event->ns		= get_pid_ns(current->nsproxy->pid_ns);
	event->id		= atomic64_inc_return(&perf_event_id);
5638

5639
	event->state		= PERF_EVENT_STATE_INACTIVE;
5640

5641 5642 5643 5644 5645 5646 5647 5648 5649 5650 5651
	if (task) {
		event->attach_state = PERF_ATTACH_TASK;
#ifdef CONFIG_HAVE_HW_BREAKPOINT
		/*
		 * hw_breakpoint is a bit difficult here..
		 */
		if (attr->type == PERF_TYPE_BREAKPOINT)
			event->hw.bp_target = task;
#endif
	}

5652
	if (!overflow_handler && parent_event) {
5653
		overflow_handler = parent_event->overflow_handler;
5654 5655
		context = parent_event->overflow_handler_context;
	}
5656

5657
	event->overflow_handler	= overflow_handler;
5658
	event->overflow_handler_context = context;
5659

5660
	if (attr->disabled)
5661
		event->state = PERF_EVENT_STATE_OFF;
5662

5663
	pmu = NULL;
5664

5665
	hwc = &event->hw;
5666
	hwc->sample_period = attr->sample_period;
5667
	if (attr->freq && attr->sample_freq)
5668
		hwc->sample_period = 1;
5669
	hwc->last_period = hwc->sample_period;
5670

5671
	local64_set(&hwc->period_left, hwc->sample_period);
5672

5673
	/*
5674
	 * we currently do not support PERF_FORMAT_GROUP on inherited events
5675
	 */
5676
	if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
5677 5678
		goto done;

5679
	pmu = perf_init_event(event);
5680

5681 5682
done:
	err = 0;
5683
	if (!pmu)
5684
		err = -EINVAL;
5685 5686
	else if (IS_ERR(pmu))
		err = PTR_ERR(pmu);
5687

5688
	if (err) {
5689 5690 5691
		if (event->ns)
			put_pid_ns(event->ns);
		kfree(event);
5692
		return ERR_PTR(err);
Ingo Molnar's avatar
Ingo Molnar committed
5693
	}
5694

5695
	if (!event->parent) {
5696
		if (event->attach_state & PERF_ATTACH_TASK)
Stephane Eranian's avatar
Stephane Eranian committed
5697
			jump_label_inc(&perf_sched_events);
5698
		if (event->attr.mmap || event->attr.mmap_data)
5699 5700 5701 5702 5703
			atomic_inc(&nr_mmap_events);
		if (event->attr.comm)
			atomic_inc(&nr_comm_events);
		if (event->attr.task)
			atomic_inc(&nr_task_events);
5704 5705 5706 5707 5708 5709 5710
		if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
			err = get_callchain_buffers();
			if (err) {
				free_event(event);
				return ERR_PTR(err);
			}
		}
5711
	}
5712

5713
	return event;
5714 5715
}

5716 5717
static int perf_copy_attr(struct perf_event_attr __user *uattr,
			  struct perf_event_attr *attr)
5718 5719
{
	u32 size;
5720
	int ret;
5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743 5744

	if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
		return -EFAULT;

	/*
	 * zero the full structure, so that a short copy will be nice.
	 */
	memset(attr, 0, sizeof(*attr));

	ret = get_user(size, &uattr->size);
	if (ret)
		return ret;

	if (size > PAGE_SIZE)	/* silly large */
		goto err_size;

	if (!size)		/* abi compat */
		size = PERF_ATTR_SIZE_VER0;

	if (size < PERF_ATTR_SIZE_VER0)
		goto err_size;

	/*
	 * If we're handed a bigger struct than we know of,
5745 5746 5747
	 * ensure all the unknown bits are 0 - i.e. new
	 * user-space does not rely on any kernel feature
	 * extensions we dont know about yet.
5748 5749
	 */
	if (size > sizeof(*attr)) {
5750 5751 5752
		unsigned char __user *addr;
		unsigned char __user *end;
		unsigned char val;
5753

5754 5755
		addr = (void __user *)uattr + sizeof(*attr);
		end  = (void __user *)uattr + size;
5756

5757
		for (; addr < end; addr++) {
5758 5759 5760 5761 5762 5763
			ret = get_user(val, addr);
			if (ret)
				return ret;
			if (val)
				goto err_size;
		}
5764
		size = sizeof(*attr);
5765 5766 5767 5768 5769 5770
	}

	ret = copy_from_user(attr, uattr, size);
	if (ret)
		return -EFAULT;

5771
	if (attr->__reserved_1)
5772 5773 5774 5775 5776 5777 5778 5779 5780 5781 5782 5783 5784 5785 5786 5787 5788
		return -EINVAL;

	if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
		return -EINVAL;

	if (attr->read_format & ~(PERF_FORMAT_MAX-1))
		return -EINVAL;

out:
	return ret;

err_size:
	put_user(sizeof(*attr), &uattr->size);
	ret = -E2BIG;
	goto out;
}

5789 5790
static int
perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
5791
{
5792
	struct ring_buffer *rb = NULL, *old_rb = NULL;
5793 5794
	int ret = -EINVAL;

5795
	if (!output_event)
5796 5797
		goto set;

5798 5799
	/* don't allow circular references */
	if (event == output_event)
5800 5801
		goto out;

5802 5803 5804 5805 5806 5807 5808
	/*
	 * Don't allow cross-cpu buffers
	 */
	if (output_event->cpu != event->cpu)
		goto out;

	/*
5809
	 * If its not a per-cpu rb, it must be the same task.
5810 5811 5812 5813
	 */
	if (output_event->cpu == -1 && output_event->ctx != event->ctx)
		goto out;

5814
set:
5815
	mutex_lock(&event->mmap_mutex);
5816 5817 5818
	/* Can't redirect output if we've got an active mmap() */
	if (atomic_read(&event->mmap_count))
		goto unlock;
5819

5820
	if (output_event) {
5821 5822 5823
		/* get the rb we want to redirect to */
		rb = ring_buffer_get(output_event);
		if (!rb)
5824
			goto unlock;
5825 5826
	}

5827 5828
	old_rb = event->rb;
	rcu_assign_pointer(event->rb, rb);
5829
	ret = 0;
5830 5831 5832
unlock:
	mutex_unlock(&event->mmap_mutex);

5833 5834
	if (old_rb)
		ring_buffer_put(old_rb);
5835 5836 5837 5838
out:
	return ret;
}

5839
/**
5840
 * sys_perf_event_open - open a performance event, associate it to a task/cpu
5841
 *
5842
 * @attr_uptr:	event_id type attributes for monitoring/sampling
5843
 * @pid:		target pid
5844
 * @cpu:		target cpu
5845
 * @group_fd:		group leader event fd
5846
 */
5847 5848
SYSCALL_DEFINE5(perf_event_open,
		struct perf_event_attr __user *, attr_uptr,
5849
		pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
5850
{
5851 5852
	struct perf_event *group_leader = NULL, *output_event = NULL;
	struct perf_event *event, *sibling;
5853 5854 5855
	struct perf_event_attr attr;
	struct perf_event_context *ctx;
	struct file *event_file = NULL;
5856
	struct file *group_file = NULL;
5857
	struct task_struct *task = NULL;
5858
	struct pmu *pmu;
5859
	int event_fd;
5860
	int move_group = 0;
5861
	int fput_needed = 0;
5862
	int err;
5863

5864
	/* for future expandability... */
Stephane Eranian's avatar
Stephane Eranian committed
5865
	if (flags & ~PERF_FLAG_ALL)
5866 5867
		return -EINVAL;

5868 5869 5870
	err = perf_copy_attr(attr_uptr, &attr);
	if (err)
		return err;
5871

5872 5873 5874 5875 5876
	if (!attr.exclude_kernel) {
		if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
			return -EACCES;
	}

5877
	if (attr.freq) {
5878
		if (attr.sample_freq > sysctl_perf_event_sample_rate)
5879 5880 5881
			return -EINVAL;
	}

Stephane Eranian's avatar
Stephane Eranian committed
5882 5883 5884 5885 5886 5887 5888 5889 5890
	/*
	 * In cgroup mode, the pid argument is used to pass the fd
	 * opened to the cgroup directory in cgroupfs. The cpu argument
	 * designates the cpu on which to monitor threads from that
	 * cgroup.
	 */
	if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
		return -EINVAL;

5891 5892 5893 5894
	event_fd = get_unused_fd_flags(O_RDWR);
	if (event_fd < 0)
		return event_fd;

5895 5896 5897 5898
	if (group_fd != -1) {
		group_leader = perf_fget_light(group_fd, &fput_needed);
		if (IS_ERR(group_leader)) {
			err = PTR_ERR(group_leader);
5899
			goto err_fd;
5900 5901 5902 5903 5904 5905 5906 5907
		}
		group_file = group_leader->filp;
		if (flags & PERF_FLAG_FD_OUTPUT)
			output_event = group_leader;
		if (flags & PERF_FLAG_FD_NO_GROUP)
			group_leader = NULL;
	}

Stephane Eranian's avatar
Stephane Eranian committed
5908
	if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
5909 5910 5911 5912 5913 5914 5915
		task = find_lively_task_by_vpid(pid);
		if (IS_ERR(task)) {
			err = PTR_ERR(task);
			goto err_group_fd;
		}
	}

5916 5917
	event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
				 NULL, NULL);
5918 5919
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
5920
		goto err_task;
5921 5922
	}

Stephane Eranian's avatar
Stephane Eranian committed
5923 5924 5925 5926
	if (flags & PERF_FLAG_PID_CGROUP) {
		err = perf_cgroup_connect(pid, event, &attr, group_leader);
		if (err)
			goto err_alloc;
5927 5928 5929 5930 5931 5932 5933
		/*
		 * one more event:
		 * - that has cgroup constraint on event->cpu
		 * - that may need work on context switch
		 */
		atomic_inc(&per_cpu(perf_cgroup_events, event->cpu));
		jump_label_inc(&perf_sched_events);
Stephane Eranian's avatar
Stephane Eranian committed
5934 5935
	}

5936 5937 5938 5939 5940
	/*
	 * Special case software events and allow them to be part of
	 * any hardware group.
	 */
	pmu = event->pmu;
5941 5942 5943 5944 5945 5946 5947 5948 5949 5950 5951 5952 5953 5954 5955 5956 5957 5958 5959 5960 5961 5962 5963

	if (group_leader &&
	    (is_software_event(event) != is_software_event(group_leader))) {
		if (is_software_event(event)) {
			/*
			 * If event and group_leader are not both a software
			 * event, and event is, then group leader is not.
			 *
			 * Allow the addition of software events to !software
			 * groups, this is safe because software events never
			 * fail to schedule.
			 */
			pmu = group_leader->pmu;
		} else if (is_software_event(group_leader) &&
			   (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
			/*
			 * In case the group is a pure software group, and we
			 * try to add a hardware event, move the whole group to
			 * the hardware context.
			 */
			move_group = 1;
		}
	}
5964 5965 5966 5967

	/*
	 * Get the target context (task or percpu):
	 */
5968
	ctx = find_get_context(pmu, task, cpu);
5969 5970
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
5971
		goto err_alloc;
5972 5973
	}

5974 5975 5976 5977 5978
	if (task) {
		put_task_struct(task);
		task = NULL;
	}

5979
	/*
5980
	 * Look up the group leader (we will attach this event to it):
5981
	 */
5982
	if (group_leader) {
5983
		err = -EINVAL;
5984 5985

		/*
5986 5987 5988 5989
		 * Do not allow a recursive hierarchy (this new sibling
		 * becoming part of another group-sibling):
		 */
		if (group_leader->group_leader != group_leader)
5990
			goto err_context;
5991 5992 5993
		/*
		 * Do not allow to attach to a group in a different
		 * task or CPU context:
5994
		 */
5995 5996 5997 5998 5999 6000 6001 6002
		if (move_group) {
			if (group_leader->ctx->type != ctx->type)
				goto err_context;
		} else {
			if (group_leader->ctx != ctx)
				goto err_context;
		}

6003 6004 6005
		/*
		 * Only a group leader can be exclusive or pinned
		 */
6006
		if (attr.exclusive || attr.pinned)
6007
			goto err_context;
6008 6009 6010 6011 6012
	}

	if (output_event) {
		err = perf_event_set_output(event, output_event);
		if (err)
6013
			goto err_context;
6014
	}
6015

6016 6017 6018
	event_file = anon_inode_getfile("[perf_event]", &perf_fops, event, O_RDWR);
	if (IS_ERR(event_file)) {
		err = PTR_ERR(event_file);
6019
		goto err_context;
6020
	}
6021

6022 6023 6024 6025
	if (move_group) {
		struct perf_event_context *gctx = group_leader->ctx;

		mutex_lock(&gctx->mutex);
6026
		perf_remove_from_context(group_leader);
6027 6028
		list_for_each_entry(sibling, &group_leader->sibling_list,
				    group_entry) {
6029
			perf_remove_from_context(sibling);
6030 6031 6032 6033
			put_ctx(gctx);
		}
		mutex_unlock(&gctx->mutex);
		put_ctx(gctx);
6034
	}
6035

6036
	event->filp = event_file;
6037
	WARN_ON_ONCE(ctx->parent_ctx);
6038
	mutex_lock(&ctx->mutex);
6039 6040 6041 6042 6043 6044 6045 6046 6047 6048 6049

	if (move_group) {
		perf_install_in_context(ctx, group_leader, cpu);
		get_ctx(ctx);
		list_for_each_entry(sibling, &group_leader->sibling_list,
				    group_entry) {
			perf_install_in_context(ctx, sibling, cpu);
			get_ctx(ctx);
		}
	}

6050
	perf_install_in_context(ctx, event, cpu);
6051
	++ctx->generation;
6052
	perf_unpin_context(ctx);
6053
	mutex_unlock(&ctx->mutex);
6054

6055
	event->owner = current;
6056

6057 6058 6059
	mutex_lock(&current->perf_event_mutex);
	list_add_tail(&event->owner_entry, &current->perf_event_list);
	mutex_unlock(&current->perf_event_mutex);
6060

6061 6062 6063 6064
	/*
	 * Precalculate sample_data sizes
	 */
	perf_event__header_size(event);
6065
	perf_event__id_header_size(event);
6066

6067 6068 6069 6070 6071 6072
	/*
	 * Drop the reference on the group_event after placing the
	 * new event on the sibling_list. This ensures destruction
	 * of the group leader will find the pointer to itself in
	 * perf_group_detach().
	 */
6073 6074 6075
	fput_light(group_file, fput_needed);
	fd_install(event_fd, event_file);
	return event_fd;
6076

6077
err_context:
6078
	perf_unpin_context(ctx);
6079
	put_ctx(ctx);
6080
err_alloc:
6081
	free_event(event);
6082 6083 6084
err_task:
	if (task)
		put_task_struct(task);
6085
err_group_fd:
6086
	fput_light(group_file, fput_needed);
6087 6088
err_fd:
	put_unused_fd(event_fd);
6089
	return err;
6090 6091
}

6092 6093 6094 6095 6096
/**
 * perf_event_create_kernel_counter
 *
 * @attr: attributes of the counter to create
 * @cpu: cpu in which the counter is bound
6097
 * @task: task to profile (NULL for percpu)
6098 6099 6100
 */
struct perf_event *
perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
6101
				 struct task_struct *task,
6102 6103
				 perf_overflow_handler_t overflow_handler,
				 void *context)
6104 6105
{
	struct perf_event_context *ctx;
6106
	struct perf_event *event;
6107
	int err;
6108

6109 6110 6111
	/*
	 * Get the target context (task or percpu):
	 */
6112

6113 6114
	event = perf_event_alloc(attr, cpu, task, NULL, NULL,
				 overflow_handler, context);
6115 6116 6117 6118
	if (IS_ERR(event)) {
		err = PTR_ERR(event);
		goto err;
	}
6119

6120
	ctx = find_get_context(event->pmu, task, cpu);
6121 6122
	if (IS_ERR(ctx)) {
		err = PTR_ERR(ctx);
6123
		goto err_free;
6124
	}
6125 6126 6127 6128 6129 6130

	event->filp = NULL;
	WARN_ON_ONCE(ctx->parent_ctx);
	mutex_lock(&ctx->mutex);
	perf_install_in_context(ctx, event, cpu);
	++ctx->generation;
6131
	perf_unpin_context(ctx);
6132 6133 6134 6135
	mutex_unlock(&ctx->mutex);

	return event;

6136 6137 6138
err_free:
	free_event(event);
err:
6139
	return ERR_PTR(err);
6140
}
6141
EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
6142

6143
static void sync_child_event(struct perf_event *child_event,
6144
			       struct task_struct *child)
6145
{
6146
	struct perf_event *parent_event = child_event->parent;
6147
	u64 child_val;
6148

6149 6150
	if (child_event->attr.inherit_stat)
		perf_event_read_event(child_event, child);
6151

6152
	child_val = perf_event_count(child_event);
6153 6154 6155 6156

	/*
	 * Add back the child's count to the parent's count:
	 */
6157
	atomic64_add(child_val, &parent_event->child_count);
6158 6159 6160 6161
	atomic64_add(child_event->total_time_enabled,
		     &parent_event->child_total_time_enabled);
	atomic64_add(child_event->total_time_running,
		     &parent_event->child_total_time_running);
6162 6163

	/*
6164
	 * Remove this event from the parent's list
6165
	 */
6166 6167 6168 6169
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_del_init(&child_event->child_list);
	mutex_unlock(&parent_event->child_mutex);
6170 6171

	/*
6172
	 * Release the parent event, if this was the last
6173 6174
	 * reference to it.
	 */
6175
	fput(parent_event->filp);
6176 6177
}

6178
static void
6179 6180
__perf_event_exit_task(struct perf_event *child_event,
			 struct perf_event_context *child_ctx,
6181
			 struct task_struct *child)
6182
{
6183 6184 6185 6186 6187
	if (child_event->parent) {
		raw_spin_lock_irq(&child_ctx->lock);
		perf_group_detach(child_event);
		raw_spin_unlock_irq(&child_ctx->lock);
	}
6188

6189
	perf_remove_from_context(child_event);
6190

6191
	/*
6192
	 * It can happen that the parent exits first, and has events
6193
	 * that are still around due to the child reference. These
6194
	 * events need to be zapped.
6195
	 */
6196
	if (child_event->parent) {
6197 6198
		sync_child_event(child_event, child);
		free_event(child_event);
6199
	}
6200 6201
}

6202
static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
6203
{
6204 6205
	struct perf_event *child_event, *tmp;
	struct perf_event_context *child_ctx;
6206
	unsigned long flags;
6207

6208
	if (likely(!child->perf_event_ctxp[ctxn])) {
6209
		perf_event_task(child, NULL, 0);
6210
		return;
6211
	}
6212

6213
	local_irq_save(flags);
6214 6215 6216 6217 6218 6219
	/*
	 * We can't reschedule here because interrupts are disabled,
	 * and either child is current or it is a task that can't be
	 * scheduled, so we are now safe from rescheduling changing
	 * our context.
	 */
6220
	child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
6221 6222 6223

	/*
	 * Take the context lock here so that if find_get_context is
6224
	 * reading child->perf_event_ctxp, we wait until it has
6225 6226
	 * incremented the context's refcount before we do put_ctx below.
	 */
6227
	raw_spin_lock(&child_ctx->lock);
6228
	task_ctx_sched_out(child_ctx);
6229
	child->perf_event_ctxp[ctxn] = NULL;
6230 6231 6232
	/*
	 * If this context is a clone; unclone it so it can't get
	 * swapped to another process while we're removing all
6233
	 * the events from it.
6234 6235
	 */
	unclone_ctx(child_ctx);
6236
	update_context_time(child_ctx);
6237
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
6238 6239

	/*
6240 6241 6242
	 * Report the task dead after unscheduling the events so that we
	 * won't get any samples after PERF_RECORD_EXIT. We can however still
	 * get a few PERF_RECORD_READ events.
6243
	 */
6244
	perf_event_task(child, child_ctx, 0);
6245

6246 6247 6248
	/*
	 * We can recurse on the same lock type through:
	 *
6249 6250 6251
	 *   __perf_event_exit_task()
	 *     sync_child_event()
	 *       fput(parent_event->filp)
6252 6253 6254 6255 6256
	 *         perf_release()
	 *           mutex_lock(&ctx->mutex)
	 *
	 * But since its the parent context it won't be the same instance.
	 */
6257
	mutex_lock(&child_ctx->mutex);
6258

6259
again:
6260 6261 6262 6263 6264
	list_for_each_entry_safe(child_event, tmp, &child_ctx->pinned_groups,
				 group_entry)
		__perf_event_exit_task(child_event, child_ctx, child);

	list_for_each_entry_safe(child_event, tmp, &child_ctx->flexible_groups,
6265
				 group_entry)
6266
		__perf_event_exit_task(child_event, child_ctx, child);
6267 6268

	/*
6269
	 * If the last event was a group event, it will have appended all
6270 6271 6272
	 * its siblings to the list, but we obtained 'tmp' before that which
	 * will still point to the list head terminating the iteration.
	 */
6273 6274
	if (!list_empty(&child_ctx->pinned_groups) ||
	    !list_empty(&child_ctx->flexible_groups))
6275
		goto again;
6276 6277 6278 6279

	mutex_unlock(&child_ctx->mutex);

	put_ctx(child_ctx);
6280 6281
}

6282 6283 6284 6285 6286
/*
 * When a child task exits, feed back event values to parent events.
 */
void perf_event_exit_task(struct task_struct *child)
{
6287
	struct perf_event *event, *tmp;
6288 6289
	int ctxn;

6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304
	mutex_lock(&child->perf_event_mutex);
	list_for_each_entry_safe(event, tmp, &child->perf_event_list,
				 owner_entry) {
		list_del_init(&event->owner_entry);

		/*
		 * Ensure the list deletion is visible before we clear
		 * the owner, closes a race against perf_release() where
		 * we need to serialize on the owner->perf_event_mutex.
		 */
		smp_wmb();
		event->owner = NULL;
	}
	mutex_unlock(&child->perf_event_mutex);

6305 6306 6307 6308
	for_each_task_context_nr(ctxn)
		perf_event_exit_task_context(child, ctxn);
}

6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321 6322
static void perf_free_event(struct perf_event *event,
			    struct perf_event_context *ctx)
{
	struct perf_event *parent = event->parent;

	if (WARN_ON_ONCE(!parent))
		return;

	mutex_lock(&parent->child_mutex);
	list_del_init(&event->child_list);
	mutex_unlock(&parent->child_mutex);

	fput(parent->filp);

6323
	perf_group_detach(event);
6324 6325 6326 6327
	list_del_event(event, ctx);
	free_event(event);
}

6328 6329
/*
 * free an unexposed, unused context as created by inheritance by
6330
 * perf_event_init_task below, used by fork() in case of fail.
6331
 */
6332
void perf_event_free_task(struct task_struct *task)
6333
{
6334
	struct perf_event_context *ctx;
6335
	struct perf_event *event, *tmp;
6336
	int ctxn;
6337

6338 6339 6340 6341
	for_each_task_context_nr(ctxn) {
		ctx = task->perf_event_ctxp[ctxn];
		if (!ctx)
			continue;
6342

6343
		mutex_lock(&ctx->mutex);
6344
again:
6345 6346 6347
		list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
				group_entry)
			perf_free_event(event, ctx);
6348

6349 6350 6351
		list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
				group_entry)
			perf_free_event(event, ctx);
6352

6353 6354 6355
		if (!list_empty(&ctx->pinned_groups) ||
				!list_empty(&ctx->flexible_groups))
			goto again;
6356

6357
		mutex_unlock(&ctx->mutex);
6358

6359 6360
		put_ctx(ctx);
	}
6361 6362
}

6363 6364 6365 6366 6367 6368 6369 6370
void perf_event_delayed_put(struct task_struct *task)
{
	int ctxn;

	for_each_task_context_nr(ctxn)
		WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
}

Peter Zijlstra's avatar
Peter Zijlstra committed
6371 6372 6373 6374 6375 6376 6377 6378 6379 6380 6381 6382
/*
 * inherit a event from parent task to child task:
 */
static struct perf_event *
inherit_event(struct perf_event *parent_event,
	      struct task_struct *parent,
	      struct perf_event_context *parent_ctx,
	      struct task_struct *child,
	      struct perf_event *group_leader,
	      struct perf_event_context *child_ctx)
{
	struct perf_event *child_event;
6383
	unsigned long flags;
Peter Zijlstra's avatar
Peter Zijlstra committed
6384 6385 6386 6387 6388 6389 6390 6391 6392 6393 6394 6395

	/*
	 * Instead of creating recursive hierarchies of events,
	 * we link inherited events back to the original parent,
	 * which has a filp for sure, which we use as the reference
	 * count:
	 */
	if (parent_event->parent)
		parent_event = parent_event->parent;

	child_event = perf_event_alloc(&parent_event->attr,
					   parent_event->cpu,
6396
					   child,
Peter Zijlstra's avatar
Peter Zijlstra committed
6397
					   group_leader, parent_event,
6398
				           NULL, NULL);
Peter Zijlstra's avatar
Peter Zijlstra committed
6399 6400 6401 6402 6403 6404 6405 6406 6407 6408 6409 6410 6411 6412 6413 6414 6415 6416 6417 6418 6419 6420 6421 6422 6423 6424
	if (IS_ERR(child_event))
		return child_event;
	get_ctx(child_ctx);

	/*
	 * Make the child state follow the state of the parent event,
	 * not its attr.disabled bit.  We hold the parent's mutex,
	 * so we won't race with perf_event_{en, dis}able_family.
	 */
	if (parent_event->state >= PERF_EVENT_STATE_INACTIVE)
		child_event->state = PERF_EVENT_STATE_INACTIVE;
	else
		child_event->state = PERF_EVENT_STATE_OFF;

	if (parent_event->attr.freq) {
		u64 sample_period = parent_event->hw.sample_period;
		struct hw_perf_event *hwc = &child_event->hw;

		hwc->sample_period = sample_period;
		hwc->last_period   = sample_period;

		local64_set(&hwc->period_left, sample_period);
	}

	child_event->ctx = child_ctx;
	child_event->overflow_handler = parent_event->overflow_handler;
6425 6426
	child_event->overflow_handler_context
		= parent_event->overflow_handler_context;
Peter Zijlstra's avatar
Peter Zijlstra committed
6427

6428 6429 6430 6431
	/*
	 * Precalculate sample_data sizes
	 */
	perf_event__header_size(child_event);
6432
	perf_event__id_header_size(child_event);
6433

Peter Zijlstra's avatar
Peter Zijlstra committed
6434 6435 6436
	/*
	 * Link it up in the child's context:
	 */
6437
	raw_spin_lock_irqsave(&child_ctx->lock, flags);
Peter Zijlstra's avatar
Peter Zijlstra committed
6438
	add_event_to_ctx(child_event, child_ctx);
6439
	raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
Peter Zijlstra's avatar
Peter Zijlstra committed
6440 6441 6442 6443 6444 6445 6446 6447 6448 6449 6450 6451 6452 6453 6454 6455 6456 6457 6458 6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480

	/*
	 * Get a reference to the parent filp - we will fput it
	 * when the child event exits. This is safe to do because
	 * we are in the parent and we know that the filp still
	 * exists and has a nonzero count:
	 */
	atomic_long_inc(&parent_event->filp->f_count);

	/*
	 * Link this into the parent event's child list
	 */
	WARN_ON_ONCE(parent_event->ctx->parent_ctx);
	mutex_lock(&parent_event->child_mutex);
	list_add_tail(&child_event->child_list, &parent_event->child_list);
	mutex_unlock(&parent_event->child_mutex);

	return child_event;
}

static int inherit_group(struct perf_event *parent_event,
	      struct task_struct *parent,
	      struct perf_event_context *parent_ctx,
	      struct task_struct *child,
	      struct perf_event_context *child_ctx)
{
	struct perf_event *leader;
	struct perf_event *sub;
	struct perf_event *child_ctr;

	leader = inherit_event(parent_event, parent, parent_ctx,
				 child, NULL, child_ctx);
	if (IS_ERR(leader))
		return PTR_ERR(leader);
	list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
		child_ctr = inherit_event(sub, parent, parent_ctx,
					    child, leader, child_ctx);
		if (IS_ERR(child_ctr))
			return PTR_ERR(child_ctr);
	}
	return 0;
6481 6482 6483 6484 6485
}

static int
inherit_task_group(struct perf_event *event, struct task_struct *parent,
		   struct perf_event_context *parent_ctx,
6486
		   struct task_struct *child, int ctxn,
6487 6488 6489
		   int *inherited_all)
{
	int ret;
6490
	struct perf_event_context *child_ctx;
6491 6492 6493 6494

	if (!event->attr.inherit) {
		*inherited_all = 0;
		return 0;
6495 6496
	}

6497
	child_ctx = child->perf_event_ctxp[ctxn];
6498 6499 6500 6501 6502 6503 6504
	if (!child_ctx) {
		/*
		 * This is executed from the parent task context, so
		 * inherit events that have been marked for cloning.
		 * First allocate and initialize a context for the
		 * child.
		 */
6505

6506
		child_ctx = alloc_perf_context(event->pmu, child);
6507 6508
		if (!child_ctx)
			return -ENOMEM;
6509

6510
		child->perf_event_ctxp[ctxn] = child_ctx;
6511 6512 6513 6514 6515 6516 6517 6518 6519
	}

	ret = inherit_group(event, parent, parent_ctx,
			    child, child_ctx);

	if (ret)
		*inherited_all = 0;

	return ret;
6520 6521
}

6522
/*
6523
 * Initialize the perf_event context in task_struct
6524
 */
6525
int perf_event_init_context(struct task_struct *child, int ctxn)
6526
{
6527
	struct perf_event_context *child_ctx, *parent_ctx;
6528 6529
	struct perf_event_context *cloned_ctx;
	struct perf_event *event;
6530
	struct task_struct *parent = current;
6531
	int inherited_all = 1;
6532
	unsigned long flags;
6533
	int ret = 0;
6534

6535
	if (likely(!parent->perf_event_ctxp[ctxn]))
6536 6537
		return 0;

6538
	/*
6539 6540
	 * If the parent's context is a clone, pin it so it won't get
	 * swapped under us.
6541
	 */
6542
	parent_ctx = perf_pin_task_context(parent, ctxn);
6543

6544 6545 6546 6547 6548 6549 6550
	/*
	 * No need to check if parent_ctx != NULL here; since we saw
	 * it non-NULL earlier, the only reason for it to become NULL
	 * is if we exit, and since we're currently in the middle of
	 * a fork we can't be exiting at the same time.
	 */

6551 6552 6553 6554
	/*
	 * Lock the parent list. No need to lock the child - not PID
	 * hashed yet and not running, so nobody can access it.
	 */
6555
	mutex_lock(&parent_ctx->mutex);
6556 6557 6558 6559 6560

	/*
	 * We dont have to disable NMIs - we are only looking at
	 * the list, not manipulating it:
	 */
6561
	list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
6562 6563
		ret = inherit_task_group(event, parent, parent_ctx,
					 child, ctxn, &inherited_all);
6564 6565 6566
		if (ret)
			break;
	}
6567

6568 6569 6570 6571 6572 6573 6574 6575 6576
	/*
	 * We can't hold ctx->lock when iterating the ->flexible_group list due
	 * to allocations, but we need to prevent rotation because
	 * rotate_ctx() will change the list from interrupt context.
	 */
	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
	parent_ctx->rotate_disable = 1;
	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);

6577
	list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
6578 6579
		ret = inherit_task_group(event, parent, parent_ctx,
					 child, ctxn, &inherited_all);
6580
		if (ret)
6581
			break;
6582 6583
	}

6584 6585 6586
	raw_spin_lock_irqsave(&parent_ctx->lock, flags);
	parent_ctx->rotate_disable = 0;

6587
	child_ctx = child->perf_event_ctxp[ctxn];
6588

6589
	if (child_ctx && inherited_all) {
6590 6591 6592
		/*
		 * Mark the child context as a clone of the parent
		 * context, or of whatever the parent is a clone of.
6593 6594 6595
		 *
		 * Note that if the parent is a clone, the holding of
		 * parent_ctx->lock avoids it from being uncloned.
6596
		 */
6597
		cloned_ctx = parent_ctx->parent_ctx;
6598 6599
		if (cloned_ctx) {
			child_ctx->parent_ctx = cloned_ctx;
6600
			child_ctx->parent_gen = parent_ctx->parent_gen;
6601 6602 6603 6604 6605
		} else {
			child_ctx->parent_ctx = parent_ctx;
			child_ctx->parent_gen = parent_ctx->generation;
		}
		get_ctx(child_ctx->parent_ctx);
6606 6607
	}

6608
	raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
6609
	mutex_unlock(&parent_ctx->mutex);
6610

6611
	perf_unpin_context(parent_ctx);
6612
	put_ctx(parent_ctx);
6613

6614
	return ret;
6615 6616
}

6617 6618 6619 6620 6621 6622 6623
/*
 * Initialize the perf_event context in task_struct
 */
int perf_event_init_task(struct task_struct *child)
{
	int ctxn, ret;

6624 6625 6626 6627
	memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
	mutex_init(&child->perf_event_mutex);
	INIT_LIST_HEAD(&child->perf_event_list);

6628 6629 6630 6631 6632 6633 6634 6635 6636
	for_each_task_context_nr(ctxn) {
		ret = perf_event_init_context(child, ctxn);
		if (ret)
			return ret;
	}

	return 0;
}

6637 6638
static void __init perf_event_init_all_cpus(void)
{
6639
	struct swevent_htable *swhash;
6640 6641 6642
	int cpu;

	for_each_possible_cpu(cpu) {
6643 6644
		swhash = &per_cpu(swevent_htable, cpu);
		mutex_init(&swhash->hlist_mutex);
6645
		INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
6646 6647 6648
	}
}

6649
static void __cpuinit perf_event_init_cpu(int cpu)
6650
{
6651
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6652

6653
	mutex_lock(&swhash->hlist_mutex);
6654
	if (swhash->hlist_refcount > 0) {
6655 6656
		struct swevent_hlist *hlist;

6657 6658 6659
		hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
		WARN_ON(!hlist);
		rcu_assign_pointer(swhash->swevent_hlist, hlist);
6660
	}
6661
	mutex_unlock(&swhash->hlist_mutex);
6662 6663
}

6664
#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
6665
static void perf_pmu_rotate_stop(struct pmu *pmu)
6666
{
6667 6668 6669 6670 6671 6672 6673
	struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);

	WARN_ON(!irqs_disabled());

	list_del_init(&cpuctx->rotation_list);
}

6674
static void __perf_event_exit_context(void *__info)
6675
{
6676
	struct perf_event_context *ctx = __info;
6677
	struct perf_event *event, *tmp;
6678

6679
	perf_pmu_rotate_stop(ctx->pmu);
6680

6681
	list_for_each_entry_safe(event, tmp, &ctx->pinned_groups, group_entry)
6682
		__perf_remove_from_context(event);
6683
	list_for_each_entry_safe(event, tmp, &ctx->flexible_groups, group_entry)
6684
		__perf_remove_from_context(event);
6685
}
6686 6687 6688 6689 6690 6691 6692 6693 6694

static void perf_event_exit_cpu_context(int cpu)
{
	struct perf_event_context *ctx;
	struct pmu *pmu;
	int idx;

	idx = srcu_read_lock(&pmus_srcu);
	list_for_each_entry_rcu(pmu, &pmus, entry) {
6695
		ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
6696 6697 6698 6699 6700 6701 6702 6703

		mutex_lock(&ctx->mutex);
		smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
		mutex_unlock(&ctx->mutex);
	}
	srcu_read_unlock(&pmus_srcu, idx);
}

6704
static void perf_event_exit_cpu(int cpu)
6705
{
6706
	struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6707

6708 6709 6710
	mutex_lock(&swhash->hlist_mutex);
	swevent_hlist_release(swhash);
	mutex_unlock(&swhash->hlist_mutex);
6711

6712
	perf_event_exit_cpu_context(cpu);
6713 6714
}
#else
6715
static inline void perf_event_exit_cpu(int cpu) { }
6716 6717
#endif

6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737
static int
perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
{
	int cpu;

	for_each_online_cpu(cpu)
		perf_event_exit_cpu(cpu);

	return NOTIFY_OK;
}

/*
 * Run the perf reboot notifier at the very last possible moment so that
 * the generic watchdog code runs as long as possible.
 */
static struct notifier_block perf_reboot_notifier = {
	.notifier_call = perf_reboot,
	.priority = INT_MIN,
};

6738 6739 6740 6741 6742
static int __cpuinit
perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
{
	unsigned int cpu = (long)hcpu;

6743
	switch (action & ~CPU_TASKS_FROZEN) {
6744 6745

	case CPU_UP_PREPARE:
Peter Zijlstra's avatar
Peter Zijlstra committed
6746
	case CPU_DOWN_FAILED:
6747
		perf_event_init_cpu(cpu);
6748 6749
		break;

Peter Zijlstra's avatar
Peter Zijlstra committed
6750
	case CPU_UP_CANCELED:
6751
	case CPU_DOWN_PREPARE:
6752
		perf_event_exit_cpu(cpu);
6753 6754 6755 6756 6757 6758 6759 6760 6761
		break;

	default:
		break;
	}

	return NOTIFY_OK;
}

6762
void __init perf_event_init(void)
6763
{
6764 6765
	int ret;

Peter Zijlstra's avatar
Peter Zijlstra committed
6766 6767
	idr_init(&pmu_idr);

6768
	perf_event_init_all_cpus();
6769
	init_srcu_struct(&pmus_srcu);
Peter Zijlstra's avatar
Peter Zijlstra committed
6770 6771 6772
	perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
	perf_pmu_register(&perf_cpu_clock, NULL, -1);
	perf_pmu_register(&perf_task_clock, NULL, -1);
6773 6774
	perf_tp_register();
	perf_cpu_notifier(perf_cpu_notify);
6775
	register_reboot_notifier(&perf_reboot_notifier);
6776 6777 6778

	ret = init_hw_breakpoint();
	WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
6779
}
Peter Zijlstra's avatar
Peter Zijlstra committed
6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792 6793 6794 6795 6796 6797 6798 6799 6800 6801 6802 6803 6804 6805 6806 6807

static int __init perf_event_sysfs_init(void)
{
	struct pmu *pmu;
	int ret;

	mutex_lock(&pmus_lock);

	ret = bus_register(&pmu_bus);
	if (ret)
		goto unlock;

	list_for_each_entry(pmu, &pmus, entry) {
		if (!pmu->name || pmu->type < 0)
			continue;

		ret = pmu_dev_alloc(pmu);
		WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
	}
	pmu_bus_running = 1;
	ret = 0;

unlock:
	mutex_unlock(&pmus_lock);

	return ret;
}
device_initcall(perf_event_sysfs_init);
Stephane Eranian's avatar
Stephane Eranian committed
6808 6809 6810 6811 6812 6813 6814

#ifdef CONFIG_CGROUP_PERF
static struct cgroup_subsys_state *perf_cgroup_create(
	struct cgroup_subsys *ss, struct cgroup *cont)
{
	struct perf_cgroup *jc;

6815
	jc = kzalloc(sizeof(*jc), GFP_KERNEL);
Stephane Eranian's avatar
Stephane Eranian committed
6816 6817 6818 6819 6820 6821 6822 6823 6824 6825 6826 6827 6828 6829 6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842 6843 6844
	if (!jc)
		return ERR_PTR(-ENOMEM);

	jc->info = alloc_percpu(struct perf_cgroup_info);
	if (!jc->info) {
		kfree(jc);
		return ERR_PTR(-ENOMEM);
	}

	return &jc->css;
}

static void perf_cgroup_destroy(struct cgroup_subsys *ss,
				struct cgroup *cont)
{
	struct perf_cgroup *jc;
	jc = container_of(cgroup_subsys_state(cont, perf_subsys_id),
			  struct perf_cgroup, css);
	free_percpu(jc->info);
	kfree(jc);
}

static int __perf_cgroup_move(void *info)
{
	struct task_struct *task = info;
	perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
	return 0;
}

6845 6846
static void
perf_cgroup_attach_task(struct cgroup *cgrp, struct task_struct *task)
Stephane Eranian's avatar
Stephane Eranian committed
6847 6848 6849 6850 6851 6852 6853 6854 6855 6856 6857 6858 6859 6860 6861
{
	task_function_call(task, __perf_cgroup_move, task);
}

static void perf_cgroup_exit(struct cgroup_subsys *ss, struct cgroup *cgrp,
		struct cgroup *old_cgrp, struct task_struct *task)
{
	/*
	 * cgroup_exit() is called in the copy_process() failure path.
	 * Ignore this case since the task hasn't ran yet, this avoids
	 * trying to poke a half freed task state from generic code.
	 */
	if (!(task->flags & PF_EXITING))
		return;

6862
	perf_cgroup_attach_task(cgrp, task);
Stephane Eranian's avatar
Stephane Eranian committed
6863 6864 6865
}

struct cgroup_subsys perf_subsys = {
6866 6867 6868 6869 6870
	.name		= "perf_event",
	.subsys_id	= perf_subsys_id,
	.create		= perf_cgroup_create,
	.destroy	= perf_cgroup_destroy,
	.exit		= perf_cgroup_exit,
6871
	.attach_task	= perf_cgroup_attach_task,
Stephane Eranian's avatar
Stephane Eranian committed
6872 6873
};
#endif /* CONFIG_CGROUP_PERF */