source: src/linux/universal/linux-3.18/kernel/events/core.c @ 31885

Last change on this file since 31885 was 31885, checked in by brainslayer, 2 months ago

update

File size: 194.2 KB
Line 
1/*
2 * Performance events core code:
3 *
4 *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 *  Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 *  Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
7 *  Copyright  ©  2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
8 *
9 * For licensing details see kernel-base/COPYING
10 */
11
12#include <linux/fs.h>
13#include <linux/mm.h>
14#include <linux/cpu.h>
15#include <linux/smp.h>
16#include <linux/idr.h>
17#include <linux/file.h>
18#include <linux/poll.h>
19#include <linux/slab.h>
20#include <linux/hash.h>
21#include <linux/tick.h>
22#include <linux/sysfs.h>
23#include <linux/dcache.h>
24#include <linux/percpu.h>
25#include <linux/ptrace.h>
26#include <linux/reboot.h>
27#include <linux/vmstat.h>
28#include <linux/device.h>
29#include <linux/export.h>
30#include <linux/vmalloc.h>
31#include <linux/hardirq.h>
32#include <linux/rculist.h>
33#include <linux/uaccess.h>
34#include <linux/syscalls.h>
35#include <linux/anon_inodes.h>
36#include <linux/kernel_stat.h>
37#include <linux/perf_event.h>
38#include <linux/ftrace_event.h>
39#include <linux/hw_breakpoint.h>
40#include <linux/mm_types.h>
41#include <linux/cgroup.h>
42#include <linux/module.h>
43#include <linux/mman.h>
44#include <linux/compat.h>
45
46#include "internal.h"
47
48#include <asm/irq_regs.h>
49
50static struct workqueue_struct *perf_wq;
51
52struct remote_function_call {
53        struct task_struct      *p;
54        int                     (*func)(void *info);
55        void                    *info;
56        int                     ret;
57};
58
59static void remote_function(void *data)
60{
61        struct remote_function_call *tfc = data;
62        struct task_struct *p = tfc->p;
63
64        if (p) {
65                tfc->ret = -EAGAIN;
66                if (task_cpu(p) != smp_processor_id() || !task_curr(p))
67                        return;
68        }
69
70        tfc->ret = tfc->func(tfc->info);
71}
72
73/**
74 * task_function_call - call a function on the cpu on which a task runs
75 * @p:          the task to evaluate
76 * @func:       the function to be called
77 * @info:       the function call argument
78 *
79 * Calls the function @func when the task is currently running. This might
80 * be on the current CPU, which just calls the function directly
81 *
82 * returns: @func return value, or
83 *          -ESRCH  - when the process isn't running
84 *          -EAGAIN - when the process moved away
85 */
86static int
87task_function_call(struct task_struct *p, int (*func) (void *info), void *info)
88{
89        struct remote_function_call data = {
90                .p      = p,
91                .func   = func,
92                .info   = info,
93                .ret    = -ESRCH, /* No such (running) process */
94        };
95
96        if (task_curr(p))
97                smp_call_function_single(task_cpu(p), remote_function, &data, 1);
98
99        return data.ret;
100}
101
102/**
103 * cpu_function_call - call a function on the cpu
104 * @func:       the function to be called
105 * @info:       the function call argument
106 *
107 * Calls the function @func on the remote cpu.
108 *
109 * returns: @func return value or -ENXIO when the cpu is offline
110 */
111static int cpu_function_call(int cpu, int (*func) (void *info), void *info)
112{
113        struct remote_function_call data = {
114                .p      = NULL,
115                .func   = func,
116                .info   = info,
117                .ret    = -ENXIO, /* No such CPU */
118        };
119
120        smp_call_function_single(cpu, remote_function, &data, 1);
121
122        return data.ret;
123}
124
125#define EVENT_OWNER_KERNEL ((void *) -1)
126
127static bool is_kernel_event(struct perf_event *event)
128{
129        return event->owner == EVENT_OWNER_KERNEL;
130}
131
132#define PERF_FLAG_ALL (PERF_FLAG_FD_NO_GROUP |\
133                       PERF_FLAG_FD_OUTPUT  |\
134                       PERF_FLAG_PID_CGROUP |\
135                       PERF_FLAG_FD_CLOEXEC)
136
137/*
138 * branch priv levels that need permission checks
139 */
140#define PERF_SAMPLE_BRANCH_PERM_PLM \
141        (PERF_SAMPLE_BRANCH_KERNEL |\
142         PERF_SAMPLE_BRANCH_HV)
143
144enum event_type_t {
145        EVENT_FLEXIBLE = 0x1,
146        EVENT_PINNED = 0x2,
147        EVENT_ALL = EVENT_FLEXIBLE | EVENT_PINNED,
148};
149
150/*
151 * perf_sched_events : >0 events exist
152 * perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
153 */
154struct static_key_deferred perf_sched_events __read_mostly;
155static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
156static DEFINE_PER_CPU(atomic_t, perf_branch_stack_events);
157
158static atomic_t nr_mmap_events __read_mostly;
159static atomic_t nr_comm_events __read_mostly;
160static atomic_t nr_task_events __read_mostly;
161static atomic_t nr_freq_events __read_mostly;
162
163static LIST_HEAD(pmus);
164static DEFINE_MUTEX(pmus_lock);
165static struct srcu_struct pmus_srcu;
166
167/*
168 * perf event paranoia level:
169 *  -1 - not paranoid at all
170 *   0 - disallow raw tracepoint access for unpriv
171 *   1 - disallow cpu events for unpriv
172 *   2 - disallow kernel profiling for unpriv
173 */
174int sysctl_perf_event_paranoid __read_mostly = 1;
175
176/* Minimum for 512 kiB + 1 user control page */
177int sysctl_perf_event_mlock __read_mostly = 512 + (PAGE_SIZE / 1024); /* 'free' kiB per user */
178
179/*
180 * max perf event sample rate
181 */
182#define DEFAULT_MAX_SAMPLE_RATE         100000
183#define DEFAULT_SAMPLE_PERIOD_NS        (NSEC_PER_SEC / DEFAULT_MAX_SAMPLE_RATE)
184#define DEFAULT_CPU_TIME_MAX_PERCENT    25
185
186int sysctl_perf_event_sample_rate __read_mostly = DEFAULT_MAX_SAMPLE_RATE;
187
188static int max_samples_per_tick __read_mostly   = DIV_ROUND_UP(DEFAULT_MAX_SAMPLE_RATE, HZ);
189static int perf_sample_period_ns __read_mostly  = DEFAULT_SAMPLE_PERIOD_NS;
190
191static int perf_sample_allowed_ns __read_mostly =
192        DEFAULT_SAMPLE_PERIOD_NS * DEFAULT_CPU_TIME_MAX_PERCENT / 100;
193
194void update_perf_cpu_limits(void)
195{
196        u64 tmp = perf_sample_period_ns;
197
198        tmp *= sysctl_perf_cpu_time_max_percent;
199        do_div(tmp, 100);
200        ACCESS_ONCE(perf_sample_allowed_ns) = tmp;
201}
202
203static int perf_rotate_context(struct perf_cpu_context *cpuctx);
204
205int perf_proc_update_handler(struct ctl_table *table, int write,
206                void __user *buffer, size_t *lenp,
207                loff_t *ppos)
208{
209        int ret = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
210
211        if (ret || !write)
212                return ret;
213
214        max_samples_per_tick = DIV_ROUND_UP(sysctl_perf_event_sample_rate, HZ);
215        perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
216        update_perf_cpu_limits();
217
218        return 0;
219}
220
221int sysctl_perf_cpu_time_max_percent __read_mostly = DEFAULT_CPU_TIME_MAX_PERCENT;
222
223int perf_cpu_time_max_percent_handler(struct ctl_table *table, int write,
224                                void __user *buffer, size_t *lenp,
225                                loff_t *ppos)
226{
227        int ret = proc_dointvec(table, write, buffer, lenp, ppos);
228
229        if (ret || !write)
230                return ret;
231
232        update_perf_cpu_limits();
233
234        return 0;
235}
236
237/*
238 * perf samples are done in some very critical code paths (NMIs).
239 * If they take too much CPU time, the system can lock up and not
240 * get any real work done.  This will drop the sample rate when
241 * we detect that events are taking too long.
242 */
243#define NR_ACCUMULATED_SAMPLES 128
244static DEFINE_PER_CPU(u64, running_sample_length);
245
246static void perf_duration_warn(struct irq_work *w)
247{
248        u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
249        u64 avg_local_sample_len;
250        u64 local_samples_len;
251
252        local_samples_len = __this_cpu_read(running_sample_length);
253        avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
254
255        printk_ratelimited(KERN_WARNING
256                        "perf interrupt took too long (%lld > %lld), lowering "
257                        "kernel.perf_event_max_sample_rate to %d\n",
258                        avg_local_sample_len, allowed_ns >> 1,
259                        sysctl_perf_event_sample_rate);
260}
261
262static DEFINE_IRQ_WORK(perf_duration_work, perf_duration_warn);
263
264void perf_sample_event_took(u64 sample_len_ns)
265{
266        u64 allowed_ns = ACCESS_ONCE(perf_sample_allowed_ns);
267        u64 avg_local_sample_len;
268        u64 local_samples_len;
269
270        if (allowed_ns == 0)
271                return;
272
273        /* decay the counter by 1 average sample */
274        local_samples_len = __this_cpu_read(running_sample_length);
275        local_samples_len -= local_samples_len/NR_ACCUMULATED_SAMPLES;
276        local_samples_len += sample_len_ns;
277        __this_cpu_write(running_sample_length, local_samples_len);
278
279        /*
280         * note: this will be biased artifically low until we have
281         * seen NR_ACCUMULATED_SAMPLES.  Doing it this way keeps us
282         * from having to maintain a count.
283         */
284        avg_local_sample_len = local_samples_len/NR_ACCUMULATED_SAMPLES;
285
286        if (avg_local_sample_len <= allowed_ns)
287                return;
288
289        if (max_samples_per_tick <= 1)
290                return;
291
292        max_samples_per_tick = DIV_ROUND_UP(max_samples_per_tick, 2);
293        sysctl_perf_event_sample_rate = max_samples_per_tick * HZ;
294        perf_sample_period_ns = NSEC_PER_SEC / sysctl_perf_event_sample_rate;
295
296        update_perf_cpu_limits();
297
298        if (!irq_work_queue(&perf_duration_work)) {
299                early_printk("perf interrupt took too long (%lld > %lld), lowering "
300                             "kernel.perf_event_max_sample_rate to %d\n",
301                             avg_local_sample_len, allowed_ns >> 1,
302                             sysctl_perf_event_sample_rate);
303        }
304}
305
306static atomic64_t perf_event_id;
307
308static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
309                              enum event_type_t event_type);
310
311static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
312                             enum event_type_t event_type,
313                             struct task_struct *task);
314
315static void update_context_time(struct perf_event_context *ctx);
316static u64 perf_event_time(struct perf_event *event);
317
318void __weak perf_event_print_debug(void)        { }
319
320extern __weak const char *perf_pmu_name(void)
321{
322        return "pmu";
323}
324
325static inline u64 perf_clock(void)
326{
327        return local_clock();
328}
329
330static inline struct perf_cpu_context *
331__get_cpu_context(struct perf_event_context *ctx)
332{
333        return this_cpu_ptr(ctx->pmu->pmu_cpu_context);
334}
335
336static void perf_ctx_lock(struct perf_cpu_context *cpuctx,
337                          struct perf_event_context *ctx)
338{
339        raw_spin_lock(&cpuctx->ctx.lock);
340        if (ctx)
341                raw_spin_lock(&ctx->lock);
342}
343
344static void perf_ctx_unlock(struct perf_cpu_context *cpuctx,
345                            struct perf_event_context *ctx)
346{
347        if (ctx)
348                raw_spin_unlock(&ctx->lock);
349        raw_spin_unlock(&cpuctx->ctx.lock);
350}
351
352#ifdef CONFIG_CGROUP_PERF
353
354/*
355 * perf_cgroup_info keeps track of time_enabled for a cgroup.
356 * This is a per-cpu dynamically allocated data structure.
357 */
358struct perf_cgroup_info {
359        u64                             time;
360        u64                             timestamp;
361};
362
363struct perf_cgroup {
364        struct cgroup_subsys_state      css;
365        struct perf_cgroup_info __percpu *info;
366};
367
368/*
369 * Must ensure cgroup is pinned (css_get) before calling
370 * this function. In other words, we cannot call this function
371 * if there is no cgroup event for the current CPU context.
372 */
373static inline struct perf_cgroup *
374perf_cgroup_from_task(struct task_struct *task)
375{
376        return container_of(task_css(task, perf_event_cgrp_id),
377                            struct perf_cgroup, css);
378}
379
380static inline bool
381perf_cgroup_match(struct perf_event *event)
382{
383        struct perf_event_context *ctx = event->ctx;
384        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
385
386        /* @event doesn't care about cgroup */
387        if (!event->cgrp)
388                return true;
389
390        /* wants specific cgroup scope but @cpuctx isn't associated with any */
391        if (!cpuctx->cgrp)
392                return false;
393
394        /*
395         * Cgroup scoping is recursive.  An event enabled for a cgroup is
396         * also enabled for all its descendant cgroups.  If @cpuctx's
397         * cgroup is a descendant of @event's (the test covers identity
398         * case), it's a match.
399         */
400        return cgroup_is_descendant(cpuctx->cgrp->css.cgroup,
401                                    event->cgrp->css.cgroup);
402}
403
404static inline void perf_detach_cgroup(struct perf_event *event)
405{
406        css_put(&event->cgrp->css);
407        event->cgrp = NULL;
408}
409
410static inline int is_cgroup_event(struct perf_event *event)
411{
412        return event->cgrp != NULL;
413}
414
415static inline u64 perf_cgroup_event_time(struct perf_event *event)
416{
417        struct perf_cgroup_info *t;
418
419        t = per_cpu_ptr(event->cgrp->info, event->cpu);
420        return t->time;
421}
422
423static inline void __update_cgrp_time(struct perf_cgroup *cgrp)
424{
425        struct perf_cgroup_info *info;
426        u64 now;
427
428        now = perf_clock();
429
430        info = this_cpu_ptr(cgrp->info);
431
432        info->time += now - info->timestamp;
433        info->timestamp = now;
434}
435
436static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
437{
438        struct perf_cgroup *cgrp_out = cpuctx->cgrp;
439        if (cgrp_out)
440                __update_cgrp_time(cgrp_out);
441}
442
443static inline void update_cgrp_time_from_event(struct perf_event *event)
444{
445        struct perf_cgroup *cgrp;
446
447        /*
448         * ensure we access cgroup data only when needed and
449         * when we know the cgroup is pinned (css_get)
450         */
451        if (!is_cgroup_event(event))
452                return;
453
454        cgrp = perf_cgroup_from_task(current);
455        /*
456         * Do not update time when cgroup is not active
457         */
458        if (cgrp == event->cgrp)
459                __update_cgrp_time(event->cgrp);
460}
461
462static inline void
463perf_cgroup_set_timestamp(struct task_struct *task,
464                          struct perf_event_context *ctx)
465{
466        struct perf_cgroup *cgrp;
467        struct perf_cgroup_info *info;
468
469        /*
470         * ctx->lock held by caller
471         * ensure we do not access cgroup data
472         * unless we have the cgroup pinned (css_get)
473         */
474        if (!task || !ctx->nr_cgroups)
475                return;
476
477        cgrp = perf_cgroup_from_task(task);
478        info = this_cpu_ptr(cgrp->info);
479        info->timestamp = ctx->timestamp;
480}
481
482#define PERF_CGROUP_SWOUT       0x1 /* cgroup switch out every event */
483#define PERF_CGROUP_SWIN        0x2 /* cgroup switch in events based on task */
484
485/*
486 * reschedule events based on the cgroup constraint of task.
487 *
488 * mode SWOUT : schedule out everything
489 * mode SWIN : schedule in based on cgroup for next
490 */
491void perf_cgroup_switch(struct task_struct *task, int mode)
492{
493        struct perf_cpu_context *cpuctx;
494        struct pmu *pmu;
495        unsigned long flags;
496
497        /*
498         * disable interrupts to avoid geting nr_cgroup
499         * changes via __perf_event_disable(). Also
500         * avoids preemption.
501         */
502        local_irq_save(flags);
503
504        /*
505         * we reschedule only in the presence of cgroup
506         * constrained events.
507         */
508        rcu_read_lock();
509
510        list_for_each_entry_rcu(pmu, &pmus, entry) {
511                cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
512                if (cpuctx->unique_pmu != pmu)
513                        continue; /* ensure we process each cpuctx once */
514
515                /*
516                 * perf_cgroup_events says at least one
517                 * context on this CPU has cgroup events.
518                 *
519                 * ctx->nr_cgroups reports the number of cgroup
520                 * events for a context.
521                 */
522                if (cpuctx->ctx.nr_cgroups > 0) {
523                        perf_ctx_lock(cpuctx, cpuctx->task_ctx);
524                        perf_pmu_disable(cpuctx->ctx.pmu);
525
526                        if (mode & PERF_CGROUP_SWOUT) {
527                                cpu_ctx_sched_out(cpuctx, EVENT_ALL);
528                                /*
529                                 * must not be done before ctxswout due
530                                 * to event_filter_match() in event_sched_out()
531                                 */
532                                cpuctx->cgrp = NULL;
533                        }
534
535                        if (mode & PERF_CGROUP_SWIN) {
536                                WARN_ON_ONCE(cpuctx->cgrp);
537                                /*
538                                 * set cgrp before ctxsw in to allow
539                                 * event_filter_match() to not have to pass
540                                 * task around
541                                 */
542                                cpuctx->cgrp = perf_cgroup_from_task(task);
543                                cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
544                        }
545                        perf_pmu_enable(cpuctx->ctx.pmu);
546                        perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
547                }
548        }
549
550        rcu_read_unlock();
551
552        local_irq_restore(flags);
553}
554
555static inline void perf_cgroup_sched_out(struct task_struct *task,
556                                         struct task_struct *next)
557{
558        struct perf_cgroup *cgrp1;
559        struct perf_cgroup *cgrp2 = NULL;
560
561        /*
562         * we come here when we know perf_cgroup_events > 0
563         */
564        cgrp1 = perf_cgroup_from_task(task);
565
566        /*
567         * next is NULL when called from perf_event_enable_on_exec()
568         * that will systematically cause a cgroup_switch()
569         */
570        if (next)
571                cgrp2 = perf_cgroup_from_task(next);
572
573        /*
574         * only schedule out current cgroup events if we know
575         * that we are switching to a different cgroup. Otherwise,
576         * do no touch the cgroup events.
577         */
578        if (cgrp1 != cgrp2)
579                perf_cgroup_switch(task, PERF_CGROUP_SWOUT);
580}
581
582static inline void perf_cgroup_sched_in(struct task_struct *prev,
583                                        struct task_struct *task)
584{
585        struct perf_cgroup *cgrp1;
586        struct perf_cgroup *cgrp2 = NULL;
587
588        /*
589         * we come here when we know perf_cgroup_events > 0
590         */
591        cgrp1 = perf_cgroup_from_task(task);
592
593        /* prev can never be NULL */
594        cgrp2 = perf_cgroup_from_task(prev);
595
596        /*
597         * only need to schedule in cgroup events if we are changing
598         * cgroup during ctxsw. Cgroup events were not scheduled
599         * out of ctxsw out if that was not the case.
600         */
601        if (cgrp1 != cgrp2)
602                perf_cgroup_switch(task, PERF_CGROUP_SWIN);
603}
604
605static inline int perf_cgroup_connect(int fd, struct perf_event *event,
606                                      struct perf_event_attr *attr,
607                                      struct perf_event *group_leader)
608{
609        struct perf_cgroup *cgrp;
610        struct cgroup_subsys_state *css;
611        struct fd f = fdget(fd);
612        int ret = 0;
613
614        if (!f.file)
615                return -EBADF;
616
617        css = css_tryget_online_from_dir(f.file->f_dentry,
618                                         &perf_event_cgrp_subsys);
619        if (IS_ERR(css)) {
620                ret = PTR_ERR(css);
621                goto out;
622        }
623
624        cgrp = container_of(css, struct perf_cgroup, css);
625        event->cgrp = cgrp;
626
627        /*
628         * all events in a group must monitor
629         * the same cgroup because a task belongs
630         * to only one perf cgroup at a time
631         */
632        if (group_leader && group_leader->cgrp != cgrp) {
633                perf_detach_cgroup(event);
634                ret = -EINVAL;
635        }
636out:
637        fdput(f);
638        return ret;
639}
640
641static inline void
642perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
643{
644        struct perf_cgroup_info *t;
645        t = per_cpu_ptr(event->cgrp->info, event->cpu);
646        event->shadow_ctx_time = now - t->timestamp;
647}
648
649static inline void
650perf_cgroup_defer_enabled(struct perf_event *event)
651{
652        /*
653         * when the current task's perf cgroup does not match
654         * the event's, we need to remember to call the
655         * perf_mark_enable() function the first time a task with
656         * a matching perf cgroup is scheduled in.
657         */
658        if (is_cgroup_event(event) && !perf_cgroup_match(event))
659                event->cgrp_defer_enabled = 1;
660}
661
662static inline void
663perf_cgroup_mark_enabled(struct perf_event *event,
664                         struct perf_event_context *ctx)
665{
666        struct perf_event *sub;
667        u64 tstamp = perf_event_time(event);
668
669        if (!event->cgrp_defer_enabled)
670                return;
671
672        event->cgrp_defer_enabled = 0;
673
674        event->tstamp_enabled = tstamp - event->total_time_enabled;
675        list_for_each_entry(sub, &event->sibling_list, group_entry) {
676                if (sub->state >= PERF_EVENT_STATE_INACTIVE) {
677                        sub->tstamp_enabled = tstamp - sub->total_time_enabled;
678                        sub->cgrp_defer_enabled = 0;
679                }
680        }
681}
682#else /* !CONFIG_CGROUP_PERF */
683
684static inline bool
685perf_cgroup_match(struct perf_event *event)
686{
687        return true;
688}
689
690static inline void perf_detach_cgroup(struct perf_event *event)
691{}
692
693static inline int is_cgroup_event(struct perf_event *event)
694{
695        return 0;
696}
697
698static inline u64 perf_cgroup_event_cgrp_time(struct perf_event *event)
699{
700        return 0;
701}
702
703static inline void update_cgrp_time_from_event(struct perf_event *event)
704{
705}
706
707static inline void update_cgrp_time_from_cpuctx(struct perf_cpu_context *cpuctx)
708{
709}
710
711static inline void perf_cgroup_sched_out(struct task_struct *task,
712                                         struct task_struct *next)
713{
714}
715
716static inline void perf_cgroup_sched_in(struct task_struct *prev,
717                                        struct task_struct *task)
718{
719}
720
721static inline int perf_cgroup_connect(pid_t pid, struct perf_event *event,
722                                      struct perf_event_attr *attr,
723                                      struct perf_event *group_leader)
724{
725        return -EINVAL;
726}
727
728static inline void
729perf_cgroup_set_timestamp(struct task_struct *task,
730                          struct perf_event_context *ctx)
731{
732}
733
734void
735perf_cgroup_switch(struct task_struct *task, struct task_struct *next)
736{
737}
738
739static inline void
740perf_cgroup_set_shadow_time(struct perf_event *event, u64 now)
741{
742}
743
744static inline u64 perf_cgroup_event_time(struct perf_event *event)
745{
746        return 0;
747}
748
749static inline void
750perf_cgroup_defer_enabled(struct perf_event *event)
751{
752}
753
754static inline void
755perf_cgroup_mark_enabled(struct perf_event *event,
756                         struct perf_event_context *ctx)
757{
758}
759#endif
760
761/*
762 * set default to be dependent on timer tick just
763 * like original code
764 */
765#define PERF_CPU_HRTIMER (1000 / HZ)
766/*
767 * function must be called with interrupts disbled
768 */
769static enum hrtimer_restart perf_cpu_hrtimer_handler(struct hrtimer *hr)
770{
771        struct perf_cpu_context *cpuctx;
772        enum hrtimer_restart ret = HRTIMER_NORESTART;
773        int rotations = 0;
774
775        WARN_ON(!irqs_disabled());
776
777        cpuctx = container_of(hr, struct perf_cpu_context, hrtimer);
778
779        rotations = perf_rotate_context(cpuctx);
780
781        /*
782         * arm timer if needed
783         */
784        if (rotations) {
785                hrtimer_forward_now(hr, cpuctx->hrtimer_interval);
786                ret = HRTIMER_RESTART;
787        }
788
789        return ret;
790}
791
792/* CPU is going down */
793void perf_cpu_hrtimer_cancel(int cpu)
794{
795        struct perf_cpu_context *cpuctx;
796        struct pmu *pmu;
797        unsigned long flags;
798
799        if (WARN_ON(cpu != smp_processor_id()))
800                return;
801
802        local_irq_save(flags);
803
804        rcu_read_lock();
805
806        list_for_each_entry_rcu(pmu, &pmus, entry) {
807                cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
808
809                if (pmu->task_ctx_nr == perf_sw_context)
810                        continue;
811
812                hrtimer_cancel(&cpuctx->hrtimer);
813        }
814
815        rcu_read_unlock();
816
817        local_irq_restore(flags);
818}
819
820static void __perf_cpu_hrtimer_init(struct perf_cpu_context *cpuctx, int cpu)
821{
822        struct hrtimer *hr = &cpuctx->hrtimer;
823        struct pmu *pmu = cpuctx->ctx.pmu;
824        int timer;
825
826        /* no multiplexing needed for SW PMU */
827        if (pmu->task_ctx_nr == perf_sw_context)
828                return;
829
830        /*
831         * check default is sane, if not set then force to
832         * default interval (1/tick)
833         */
834        timer = pmu->hrtimer_interval_ms;
835        if (timer < 1)
836                timer = pmu->hrtimer_interval_ms = PERF_CPU_HRTIMER;
837
838        cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
839
840        hrtimer_init(hr, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
841        hr->function = perf_cpu_hrtimer_handler;
842}
843
844static void perf_cpu_hrtimer_restart(struct perf_cpu_context *cpuctx)
845{
846        struct hrtimer *hr = &cpuctx->hrtimer;
847        struct pmu *pmu = cpuctx->ctx.pmu;
848
849        /* not for SW PMU */
850        if (pmu->task_ctx_nr == perf_sw_context)
851                return;
852
853        if (hrtimer_active(hr))
854                return;
855
856        if (!hrtimer_callback_running(hr))
857                __hrtimer_start_range_ns(hr, cpuctx->hrtimer_interval,
858                                         0, HRTIMER_MODE_REL_PINNED, 0);
859}
860
861void perf_pmu_disable(struct pmu *pmu)
862{
863        int *count = this_cpu_ptr(pmu->pmu_disable_count);
864        if (!(*count)++)
865                pmu->pmu_disable(pmu);
866}
867
868void perf_pmu_enable(struct pmu *pmu)
869{
870        int *count = this_cpu_ptr(pmu->pmu_disable_count);
871        if (!--(*count))
872                pmu->pmu_enable(pmu);
873}
874
875static DEFINE_PER_CPU(struct list_head, rotation_list);
876
877/*
878 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
879 * because they're strictly cpu affine and rotate_start is called with IRQs
880 * disabled, while rotate_context is called from IRQ context.
881 */
882static void perf_pmu_rotate_start(struct pmu *pmu)
883{
884        struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
885        struct list_head *head = this_cpu_ptr(&rotation_list);
886
887        WARN_ON(!irqs_disabled());
888
889        if (list_empty(&cpuctx->rotation_list))
890                list_add(&cpuctx->rotation_list, head);
891}
892
893static void get_ctx(struct perf_event_context *ctx)
894{
895        WARN_ON(!atomic_inc_not_zero(&ctx->refcount));
896}
897
898static void put_ctx(struct perf_event_context *ctx)
899{
900        if (atomic_dec_and_test(&ctx->refcount)) {
901                if (ctx->parent_ctx)
902                        put_ctx(ctx->parent_ctx);
903                if (ctx->task)
904                        put_task_struct(ctx->task);
905                kfree_rcu(ctx, rcu_head);
906        }
907}
908
909/*
910 * This must be done under the ctx->lock, such as to serialize against
911 * context_equiv(), therefore we cannot call put_ctx() since that might end up
912 * calling scheduler related locks and ctx->lock nests inside those.
913 */
914static __must_check struct perf_event_context *
915unclone_ctx(struct perf_event_context *ctx)
916{
917        struct perf_event_context *parent_ctx = ctx->parent_ctx;
918
919        lockdep_assert_held(&ctx->lock);
920
921        if (parent_ctx)
922                ctx->parent_ctx = NULL;
923        ctx->generation++;
924
925        return parent_ctx;
926}
927
928static u32 perf_event_pid(struct perf_event *event, struct task_struct *p)
929{
930        /*
931         * only top level events have the pid namespace they were created in
932         */
933        if (event->parent)
934                event = event->parent;
935
936        return task_tgid_nr_ns(p, event->ns);
937}
938
939static u32 perf_event_tid(struct perf_event *event, struct task_struct *p)
940{
941        /*
942         * only top level events have the pid namespace they were created in
943         */
944        if (event->parent)
945                event = event->parent;
946
947        return task_pid_nr_ns(p, event->ns);
948}
949
950/*
951 * If we inherit events we want to return the parent event id
952 * to userspace.
953 */
954static u64 primary_event_id(struct perf_event *event)
955{
956        u64 id = event->id;
957
958        if (event->parent)
959                id = event->parent->id;
960
961        return id;
962}
963
964/*
965 * Get the perf_event_context for a task and lock it.
966 * This has to cope with with the fact that until it is locked,
967 * the context could get moved to another task.
968 */
969static struct perf_event_context *
970perf_lock_task_context(struct task_struct *task, int ctxn, unsigned long *flags)
971{
972        struct perf_event_context *ctx;
973
974retry:
975        /*
976         * One of the few rules of preemptible RCU is that one cannot do
977         * rcu_read_unlock() while holding a scheduler (or nested) lock when
978         * part of the read side critical section was preemptible -- see
979         * rcu_read_unlock_special().
980         *
981         * Since ctx->lock nests under rq->lock we must ensure the entire read
982         * side critical section is non-preemptible.
983         */
984        preempt_disable();
985        rcu_read_lock();
986        ctx = rcu_dereference(task->perf_event_ctxp[ctxn]);
987        if (ctx) {
988                /*
989                 * If this context is a clone of another, it might
990                 * get swapped for another underneath us by
991                 * perf_event_task_sched_out, though the
992                 * rcu_read_lock() protects us from any context
993                 * getting freed.  Lock the context and check if it
994                 * got swapped before we could get the lock, and retry
995                 * if so.  If we locked the right context, then it
996                 * can't get swapped on us any more.
997                 */
998                raw_spin_lock_irqsave(&ctx->lock, *flags);
999                if (ctx != rcu_dereference(task->perf_event_ctxp[ctxn])) {
1000                        raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1001                        rcu_read_unlock();
1002                        preempt_enable();
1003                        goto retry;
1004                }
1005
1006                if (!atomic_inc_not_zero(&ctx->refcount)) {
1007                        raw_spin_unlock_irqrestore(&ctx->lock, *flags);
1008                        ctx = NULL;
1009                }
1010        }
1011        rcu_read_unlock();
1012        preempt_enable();
1013        return ctx;
1014}
1015
1016/*
1017 * Get the context for a task and increment its pin_count so it
1018 * can't get swapped to another task.  This also increments its
1019 * reference count so that the context can't get freed.
1020 */
1021static struct perf_event_context *
1022perf_pin_task_context(struct task_struct *task, int ctxn)
1023{
1024        struct perf_event_context *ctx;
1025        unsigned long flags;
1026
1027        ctx = perf_lock_task_context(task, ctxn, &flags);
1028        if (ctx) {
1029                ++ctx->pin_count;
1030                raw_spin_unlock_irqrestore(&ctx->lock, flags);
1031        }
1032        return ctx;
1033}
1034
1035static void perf_unpin_context(struct perf_event_context *ctx)
1036{
1037        unsigned long flags;
1038
1039        raw_spin_lock_irqsave(&ctx->lock, flags);
1040        --ctx->pin_count;
1041        raw_spin_unlock_irqrestore(&ctx->lock, flags);
1042}
1043
1044/*
1045 * Update the record of the current time in a context.
1046 */
1047static void update_context_time(struct perf_event_context *ctx)
1048{
1049        u64 now = perf_clock();
1050
1051        ctx->time += now - ctx->timestamp;
1052        ctx->timestamp = now;
1053}
1054
1055static u64 perf_event_time(struct perf_event *event)
1056{
1057        struct perf_event_context *ctx = event->ctx;
1058
1059        if (is_cgroup_event(event))
1060                return perf_cgroup_event_time(event);
1061
1062        return ctx ? ctx->time : 0;
1063}
1064
1065/*
1066 * Update the total_time_enabled and total_time_running fields for a event.
1067 * The caller of this function needs to hold the ctx->lock.
1068 */
1069static void update_event_times(struct perf_event *event)
1070{
1071        struct perf_event_context *ctx = event->ctx;
1072        u64 run_end;
1073
1074        if (event->state < PERF_EVENT_STATE_INACTIVE ||
1075            event->group_leader->state < PERF_EVENT_STATE_INACTIVE)
1076                return;
1077        /*
1078         * in cgroup mode, time_enabled represents
1079         * the time the event was enabled AND active
1080         * tasks were in the monitored cgroup. This is
1081         * independent of the activity of the context as
1082         * there may be a mix of cgroup and non-cgroup events.
1083         *
1084         * That is why we treat cgroup events differently
1085         * here.
1086         */
1087        if (is_cgroup_event(event))
1088                run_end = perf_cgroup_event_time(event);
1089        else if (ctx->is_active)
1090                run_end = ctx->time;
1091        else
1092                run_end = event->tstamp_stopped;
1093
1094        event->total_time_enabled = run_end - event->tstamp_enabled;
1095
1096        if (event->state == PERF_EVENT_STATE_INACTIVE)
1097                run_end = event->tstamp_stopped;
1098        else
1099                run_end = perf_event_time(event);
1100
1101        event->total_time_running = run_end - event->tstamp_running;
1102
1103}
1104
1105/*
1106 * Update total_time_enabled and total_time_running for all events in a group.
1107 */
1108static void update_group_times(struct perf_event *leader)
1109{
1110        struct perf_event *event;
1111
1112        update_event_times(leader);
1113        list_for_each_entry(event, &leader->sibling_list, group_entry)
1114                update_event_times(event);
1115}
1116
1117static struct list_head *
1118ctx_group_list(struct perf_event *event, struct perf_event_context *ctx)
1119{
1120        if (event->attr.pinned)
1121                return &ctx->pinned_groups;
1122        else
1123                return &ctx->flexible_groups;
1124}
1125
1126/*
1127 * Add a event from the lists for its context.
1128 * Must be called with ctx->mutex and ctx->lock held.
1129 */
1130static void
1131list_add_event(struct perf_event *event, struct perf_event_context *ctx)
1132{
1133        WARN_ON_ONCE(event->attach_state & PERF_ATTACH_CONTEXT);
1134        event->attach_state |= PERF_ATTACH_CONTEXT;
1135
1136        /*
1137         * If we're a stand alone event or group leader, we go to the context
1138         * list, group events are kept attached to the group so that
1139         * perf_group_detach can, at all times, locate all siblings.
1140         */
1141        if (event->group_leader == event) {
1142                struct list_head *list;
1143
1144                if (is_software_event(event))
1145                        event->group_flags |= PERF_GROUP_SOFTWARE;
1146
1147                list = ctx_group_list(event, ctx);
1148                list_add_tail(&event->group_entry, list);
1149        }
1150
1151        if (is_cgroup_event(event))
1152                ctx->nr_cgroups++;
1153
1154        if (has_branch_stack(event))
1155                ctx->nr_branch_stack++;
1156
1157        list_add_rcu(&event->event_entry, &ctx->event_list);
1158        if (!ctx->nr_events)
1159                perf_pmu_rotate_start(ctx->pmu);
1160        ctx->nr_events++;
1161        if (event->attr.inherit_stat)
1162                ctx->nr_stat++;
1163
1164        ctx->generation++;
1165}
1166
1167/*
1168 * Initialize event state based on the perf_event_attr::disabled.
1169 */
1170static inline void perf_event__state_init(struct perf_event *event)
1171{
1172        event->state = event->attr.disabled ? PERF_EVENT_STATE_OFF :
1173                                              PERF_EVENT_STATE_INACTIVE;
1174}
1175
1176/*
1177 * Called at perf_event creation and when events are attached/detached from a
1178 * group.
1179 */
1180static void perf_event__read_size(struct perf_event *event)
1181{
1182        int entry = sizeof(u64); /* value */
1183        int size = 0;
1184        int nr = 1;
1185
1186        if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
1187                size += sizeof(u64);
1188
1189        if (event->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
1190                size += sizeof(u64);
1191
1192        if (event->attr.read_format & PERF_FORMAT_ID)
1193                entry += sizeof(u64);
1194
1195        if (event->attr.read_format & PERF_FORMAT_GROUP) {
1196                nr += event->group_leader->nr_siblings;
1197                size += sizeof(u64);
1198        }
1199
1200        size += entry * nr;
1201        event->read_size = size;
1202}
1203
1204static void perf_event__header_size(struct perf_event *event)
1205{
1206        struct perf_sample_data *data;
1207        u64 sample_type = event->attr.sample_type;
1208        u16 size = 0;
1209
1210        perf_event__read_size(event);
1211
1212        if (sample_type & PERF_SAMPLE_IP)
1213                size += sizeof(data->ip);
1214
1215        if (sample_type & PERF_SAMPLE_ADDR)
1216                size += sizeof(data->addr);
1217
1218        if (sample_type & PERF_SAMPLE_PERIOD)
1219                size += sizeof(data->period);
1220
1221        if (sample_type & PERF_SAMPLE_WEIGHT)
1222                size += sizeof(data->weight);
1223
1224        if (sample_type & PERF_SAMPLE_READ)
1225                size += event->read_size;
1226
1227        if (sample_type & PERF_SAMPLE_DATA_SRC)
1228                size += sizeof(data->data_src.val);
1229
1230        if (sample_type & PERF_SAMPLE_TRANSACTION)
1231                size += sizeof(data->txn);
1232
1233        event->header_size = size;
1234}
1235
1236static void perf_event__id_header_size(struct perf_event *event)
1237{
1238        struct perf_sample_data *data;
1239        u64 sample_type = event->attr.sample_type;
1240        u16 size = 0;
1241
1242        if (sample_type & PERF_SAMPLE_TID)
1243                size += sizeof(data->tid_entry);
1244
1245        if (sample_type & PERF_SAMPLE_TIME)
1246                size += sizeof(data->time);
1247
1248        if (sample_type & PERF_SAMPLE_IDENTIFIER)
1249                size += sizeof(data->id);
1250
1251        if (sample_type & PERF_SAMPLE_ID)
1252                size += sizeof(data->id);
1253
1254        if (sample_type & PERF_SAMPLE_STREAM_ID)
1255                size += sizeof(data->stream_id);
1256
1257        if (sample_type & PERF_SAMPLE_CPU)
1258                size += sizeof(data->cpu_entry);
1259
1260        event->id_header_size = size;
1261}
1262
1263static void perf_group_attach(struct perf_event *event)
1264{
1265        struct perf_event *group_leader = event->group_leader, *pos;
1266
1267        /*
1268         * We can have double attach due to group movement in perf_event_open.
1269         */
1270        if (event->attach_state & PERF_ATTACH_GROUP)
1271                return;
1272
1273        event->attach_state |= PERF_ATTACH_GROUP;
1274
1275        if (group_leader == event)
1276                return;
1277
1278        if (group_leader->group_flags & PERF_GROUP_SOFTWARE &&
1279                        !is_software_event(event))
1280                group_leader->group_flags &= ~PERF_GROUP_SOFTWARE;
1281
1282        list_add_tail(&event->group_entry, &group_leader->sibling_list);
1283        group_leader->nr_siblings++;
1284
1285        perf_event__header_size(group_leader);
1286
1287        list_for_each_entry(pos, &group_leader->sibling_list, group_entry)
1288                perf_event__header_size(pos);
1289}
1290
1291/*
1292 * Remove a event from the lists for its context.
1293 * Must be called with ctx->mutex and ctx->lock held.
1294 */
1295static void
1296list_del_event(struct perf_event *event, struct perf_event_context *ctx)
1297{
1298        struct perf_cpu_context *cpuctx;
1299        /*
1300         * We can have double detach due to exit/hot-unplug + close.
1301         */
1302        if (!(event->attach_state & PERF_ATTACH_CONTEXT))
1303                return;
1304
1305        event->attach_state &= ~PERF_ATTACH_CONTEXT;
1306
1307        if (is_cgroup_event(event)) {
1308                ctx->nr_cgroups--;
1309                cpuctx = __get_cpu_context(ctx);
1310                /*
1311                 * if there are no more cgroup events
1312                 * then cler cgrp to avoid stale pointer
1313                 * in update_cgrp_time_from_cpuctx()
1314                 */
1315                if (!ctx->nr_cgroups)
1316                        cpuctx->cgrp = NULL;
1317        }
1318
1319        if (has_branch_stack(event))
1320                ctx->nr_branch_stack--;
1321
1322        ctx->nr_events--;
1323        if (event->attr.inherit_stat)
1324                ctx->nr_stat--;
1325
1326        list_del_rcu(&event->event_entry);
1327
1328        if (event->group_leader == event)
1329                list_del_init(&event->group_entry);
1330
1331        update_group_times(event);
1332
1333        /*
1334         * If event was in error state, then keep it
1335         * that way, otherwise bogus counts will be
1336         * returned on read(). The only way to get out
1337         * of error state is by explicit re-enabling
1338         * of the event
1339         */
1340        if (event->state > PERF_EVENT_STATE_OFF)
1341                event->state = PERF_EVENT_STATE_OFF;
1342
1343        ctx->generation++;
1344}
1345
1346static void perf_group_detach(struct perf_event *event)
1347{
1348        struct perf_event *sibling, *tmp;
1349        struct list_head *list = NULL;
1350
1351        /*
1352         * We can have double detach due to exit/hot-unplug + close.
1353         */
1354        if (!(event->attach_state & PERF_ATTACH_GROUP))
1355                return;
1356
1357        event->attach_state &= ~PERF_ATTACH_GROUP;
1358
1359        /*
1360         * If this is a sibling, remove it from its group.
1361         */
1362        if (event->group_leader != event) {
1363                list_del_init(&event->group_entry);
1364                event->group_leader->nr_siblings--;
1365                goto out;
1366        }
1367
1368        if (!list_empty(&event->group_entry))
1369                list = &event->group_entry;
1370
1371        /*
1372         * If this was a group event with sibling events then
1373         * upgrade the siblings to singleton events by adding them
1374         * to whatever list we are on.
1375         */
1376        list_for_each_entry_safe(sibling, tmp, &event->sibling_list, group_entry) {
1377                if (list)
1378                        list_move_tail(&sibling->group_entry, list);
1379                sibling->group_leader = sibling;
1380
1381                /* Inherit group flags from the previous leader */
1382                sibling->group_flags = event->group_flags;
1383        }
1384
1385out:
1386        perf_event__header_size(event->group_leader);
1387
1388        list_for_each_entry(tmp, &event->group_leader->sibling_list, group_entry)
1389                perf_event__header_size(tmp);
1390}
1391
1392/*
1393 * User event without the task.
1394 */
1395static bool is_orphaned_event(struct perf_event *event)
1396{
1397        return event && !is_kernel_event(event) && !event->owner;
1398}
1399
1400/*
1401 * Event has a parent but parent's task finished and it's
1402 * alive only because of children holding refference.
1403 */
1404static bool is_orphaned_child(struct perf_event *event)
1405{
1406        return is_orphaned_event(event->parent);
1407}
1408
1409static void orphans_remove_work(struct work_struct *work);
1410
1411static void schedule_orphans_remove(struct perf_event_context *ctx)
1412{
1413        if (!ctx->task || ctx->orphans_remove_sched || !perf_wq)
1414                return;
1415
1416        if (queue_delayed_work(perf_wq, &ctx->orphans_remove, 1)) {
1417                get_ctx(ctx);
1418                ctx->orphans_remove_sched = true;
1419        }
1420}
1421
1422static int __init perf_workqueue_init(void)
1423{
1424        perf_wq = create_singlethread_workqueue("perf");
1425        WARN(!perf_wq, "failed to create perf workqueue\n");
1426        return perf_wq ? 0 : -1;
1427}
1428
1429core_initcall(perf_workqueue_init);
1430
1431static inline int
1432event_filter_match(struct perf_event *event)
1433{
1434        return (event->cpu == -1 || event->cpu == smp_processor_id())
1435            && perf_cgroup_match(event);
1436}
1437
1438static void
1439event_sched_out(struct perf_event *event,
1440                  struct perf_cpu_context *cpuctx,
1441                  struct perf_event_context *ctx)
1442{
1443        u64 tstamp = perf_event_time(event);
1444        u64 delta;
1445        /*
1446         * An event which could not be activated because of
1447         * filter mismatch still needs to have its timings
1448         * maintained, otherwise bogus information is return
1449         * via read() for time_enabled, time_running:
1450         */
1451        if (event->state == PERF_EVENT_STATE_INACTIVE
1452            && !event_filter_match(event)) {
1453                delta = tstamp - event->tstamp_stopped;
1454                event->tstamp_running += delta;
1455                event->tstamp_stopped = tstamp;
1456        }
1457
1458        if (event->state != PERF_EVENT_STATE_ACTIVE)
1459                return;
1460
1461        perf_pmu_disable(event->pmu);
1462
1463        event->state = PERF_EVENT_STATE_INACTIVE;
1464        if (event->pending_disable) {
1465                event->pending_disable = 0;
1466                event->state = PERF_EVENT_STATE_OFF;
1467        }
1468        event->tstamp_stopped = tstamp;
1469        event->pmu->del(event, 0);
1470        event->oncpu = -1;
1471
1472        if (!is_software_event(event))
1473                cpuctx->active_oncpu--;
1474        ctx->nr_active--;
1475        if (event->attr.freq && event->attr.sample_freq)
1476                ctx->nr_freq--;
1477        if (event->attr.exclusive || !cpuctx->active_oncpu)
1478                cpuctx->exclusive = 0;
1479
1480        if (is_orphaned_child(event))
1481                schedule_orphans_remove(ctx);
1482
1483        perf_pmu_enable(event->pmu);
1484}
1485
1486static void
1487group_sched_out(struct perf_event *group_event,
1488                struct perf_cpu_context *cpuctx,
1489                struct perf_event_context *ctx)
1490{
1491        struct perf_event *event;
1492        int state = group_event->state;
1493
1494        event_sched_out(group_event, cpuctx, ctx);
1495
1496        /*
1497         * Schedule out siblings (if any):
1498         */
1499        list_for_each_entry(event, &group_event->sibling_list, group_entry)
1500                event_sched_out(event, cpuctx, ctx);
1501
1502        if (state == PERF_EVENT_STATE_ACTIVE && group_event->attr.exclusive)
1503                cpuctx->exclusive = 0;
1504}
1505
1506struct remove_event {
1507        struct perf_event *event;
1508        bool detach_group;
1509};
1510
1511/*
1512 * Cross CPU call to remove a performance event
1513 *
1514 * We disable the event on the hardware level first. After that we
1515 * remove it from the context list.
1516 */
1517static int __perf_remove_from_context(void *info)
1518{
1519        struct remove_event *re = info;
1520        struct perf_event *event = re->event;
1521        struct perf_event_context *ctx = event->ctx;
1522        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1523
1524        raw_spin_lock(&ctx->lock);
1525        event_sched_out(event, cpuctx, ctx);
1526        if (re->detach_group)
1527                perf_group_detach(event);
1528        list_del_event(event, ctx);
1529        if (!ctx->nr_events && cpuctx->task_ctx == ctx) {
1530                ctx->is_active = 0;
1531                cpuctx->task_ctx = NULL;
1532        }
1533        raw_spin_unlock(&ctx->lock);
1534
1535        return 0;
1536}
1537
1538
1539/*
1540 * Remove the event from a task's (or a CPU's) list of events.
1541 *
1542 * CPU events are removed with a smp call. For task events we only
1543 * call when the task is on a CPU.
1544 *
1545 * If event->ctx is a cloned context, callers must make sure that
1546 * every task struct that event->ctx->task could possibly point to
1547 * remains valid.  This is OK when called from perf_release since
1548 * that only calls us on the top-level context, which can't be a clone.
1549 * When called from perf_event_exit_task, it's OK because the
1550 * context has been detached from its task.
1551 */
1552static void perf_remove_from_context(struct perf_event *event, bool detach_group)
1553{
1554        struct perf_event_context *ctx = event->ctx;
1555        struct task_struct *task = ctx->task;
1556        struct remove_event re = {
1557                .event = event,
1558                .detach_group = detach_group,
1559        };
1560
1561        lockdep_assert_held(&ctx->mutex);
1562
1563        if (!task) {
1564                /*
1565                 * Per cpu events are removed via an smp call. The removal can
1566                 * fail if the CPU is currently offline, but in that case we
1567                 * already called __perf_remove_from_context from
1568                 * perf_event_exit_cpu.
1569                 */
1570                cpu_function_call(event->cpu, __perf_remove_from_context, &re);
1571                return;
1572        }
1573
1574retry:
1575        if (!task_function_call(task, __perf_remove_from_context, &re))
1576                return;
1577
1578        raw_spin_lock_irq(&ctx->lock);
1579        /*
1580         * If we failed to find a running task, but find the context active now
1581         * that we've acquired the ctx->lock, retry.
1582         */
1583        if (ctx->is_active) {
1584                raw_spin_unlock_irq(&ctx->lock);
1585                /*
1586                 * Reload the task pointer, it might have been changed by
1587                 * a concurrent perf_event_context_sched_out().
1588                 */
1589                task = ctx->task;
1590                goto retry;
1591        }
1592
1593        /*
1594         * Since the task isn't running, its safe to remove the event, us
1595         * holding the ctx->lock ensures the task won't get scheduled in.
1596         */
1597        if (detach_group)
1598                perf_group_detach(event);
1599        list_del_event(event, ctx);
1600        raw_spin_unlock_irq(&ctx->lock);
1601}
1602
1603/*
1604 * Cross CPU call to disable a performance event
1605 */
1606int __perf_event_disable(void *info)
1607{
1608        struct perf_event *event = info;
1609        struct perf_event_context *ctx = event->ctx;
1610        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1611
1612        /*
1613         * If this is a per-task event, need to check whether this
1614         * event's task is the current task on this cpu.
1615         *
1616         * Can trigger due to concurrent perf_event_context_sched_out()
1617         * flipping contexts around.
1618         */
1619        if (ctx->task && cpuctx->task_ctx != ctx)
1620                return -EINVAL;
1621
1622        raw_spin_lock(&ctx->lock);
1623
1624        /*
1625         * If the event is on, turn it off.
1626         * If it is in error state, leave it in error state.
1627         */
1628        if (event->state >= PERF_EVENT_STATE_INACTIVE) {
1629                update_context_time(ctx);
1630                update_cgrp_time_from_event(event);
1631                update_group_times(event);
1632                if (event == event->group_leader)
1633                        group_sched_out(event, cpuctx, ctx);
1634                else
1635                        event_sched_out(event, cpuctx, ctx);
1636                event->state = PERF_EVENT_STATE_OFF;
1637        }
1638
1639        raw_spin_unlock(&ctx->lock);
1640
1641        return 0;
1642}
1643
1644/*
1645 * Disable a event.
1646 *
1647 * If event->ctx is a cloned context, callers must make sure that
1648 * every task struct that event->ctx->task could possibly point to
1649 * remains valid.  This condition is satisifed when called through
1650 * perf_event_for_each_child or perf_event_for_each because they
1651 * hold the top-level event's child_mutex, so any descendant that
1652 * goes to exit will block in sync_child_event.
1653 * When called from perf_pending_event it's OK because event->ctx
1654 * is the current context on this CPU and preemption is disabled,
1655 * hence we can't get into perf_event_task_sched_out for this context.
1656 */
1657void perf_event_disable(struct perf_event *event)
1658{
1659        struct perf_event_context *ctx = event->ctx;
1660        struct task_struct *task = ctx->task;
1661
1662        if (!task) {
1663                /*
1664                 * Disable the event on the cpu that it's on
1665                 */
1666                cpu_function_call(event->cpu, __perf_event_disable, event);
1667                return;
1668        }
1669
1670retry:
1671        if (!task_function_call(task, __perf_event_disable, event))
1672                return;
1673
1674        raw_spin_lock_irq(&ctx->lock);
1675        /*
1676         * If the event is still active, we need to retry the cross-call.
1677         */
1678        if (event->state == PERF_EVENT_STATE_ACTIVE) {
1679                raw_spin_unlock_irq(&ctx->lock);
1680                /*
1681                 * Reload the task pointer, it might have been changed by
1682                 * a concurrent perf_event_context_sched_out().
1683                 */
1684                task = ctx->task;
1685                goto retry;
1686        }
1687
1688        /*
1689         * Since we have the lock this context can't be scheduled
1690         * in, so we can change the state safely.
1691         */
1692        if (event->state == PERF_EVENT_STATE_INACTIVE) {
1693                update_group_times(event);
1694                event->state = PERF_EVENT_STATE_OFF;
1695        }
1696        raw_spin_unlock_irq(&ctx->lock);
1697}
1698EXPORT_SYMBOL_GPL(perf_event_disable);
1699
1700static void perf_set_shadow_time(struct perf_event *event,
1701                                 struct perf_event_context *ctx,
1702                                 u64 tstamp)
1703{
1704        /*
1705         * use the correct time source for the time snapshot
1706         *
1707         * We could get by without this by leveraging the
1708         * fact that to get to this function, the caller
1709         * has most likely already called update_context_time()
1710         * and update_cgrp_time_xx() and thus both timestamp
1711         * are identical (or very close). Given that tstamp is,
1712         * already adjusted for cgroup, we could say that:
1713         *    tstamp - ctx->timestamp
1714         * is equivalent to
1715         *    tstamp - cgrp->timestamp.
1716         *
1717         * Then, in perf_output_read(), the calculation would
1718         * work with no changes because:
1719         * - event is guaranteed scheduled in
1720         * - no scheduled out in between
1721         * - thus the timestamp would be the same
1722         *
1723         * But this is a bit hairy.
1724         *
1725         * So instead, we have an explicit cgroup call to remain
1726         * within the time time source all along. We believe it
1727         * is cleaner and simpler to understand.
1728         */
1729        if (is_cgroup_event(event))
1730                perf_cgroup_set_shadow_time(event, tstamp);
1731        else
1732                event->shadow_ctx_time = tstamp - ctx->timestamp;
1733}
1734
1735#define MAX_INTERRUPTS (~0ULL)
1736
1737static void perf_log_throttle(struct perf_event *event, int enable);
1738
1739static int
1740event_sched_in(struct perf_event *event,
1741                 struct perf_cpu_context *cpuctx,
1742                 struct perf_event_context *ctx)
1743{
1744        u64 tstamp = perf_event_time(event);
1745        int ret = 0;
1746
1747        lockdep_assert_held(&ctx->lock);
1748
1749        if (event->state <= PERF_EVENT_STATE_OFF)
1750                return 0;
1751
1752        event->state = PERF_EVENT_STATE_ACTIVE;
1753        event->oncpu = smp_processor_id();
1754
1755        /*
1756         * Unthrottle events, since we scheduled we might have missed several
1757         * ticks already, also for a heavily scheduling task there is little
1758         * guarantee it'll get a tick in a timely manner.
1759         */
1760        if (unlikely(event->hw.interrupts == MAX_INTERRUPTS)) {
1761                perf_log_throttle(event, 1);
1762                event->hw.interrupts = 0;
1763        }
1764
1765        /*
1766         * The new state must be visible before we turn it on in the hardware:
1767         */
1768        smp_wmb();
1769
1770        perf_pmu_disable(event->pmu);
1771
1772        if (event->pmu->add(event, PERF_EF_START)) {
1773                event->state = PERF_EVENT_STATE_INACTIVE;
1774                event->oncpu = -1;
1775                ret = -EAGAIN;
1776                goto out;
1777        }
1778
1779        event->tstamp_running += tstamp - event->tstamp_stopped;
1780
1781        perf_set_shadow_time(event, ctx, tstamp);
1782
1783        if (!is_software_event(event))
1784                cpuctx->active_oncpu++;
1785        ctx->nr_active++;
1786        if (event->attr.freq && event->attr.sample_freq)
1787                ctx->nr_freq++;
1788
1789        if (event->attr.exclusive)
1790                cpuctx->exclusive = 1;
1791
1792        if (is_orphaned_child(event))
1793                schedule_orphans_remove(ctx);
1794
1795out:
1796        perf_pmu_enable(event->pmu);
1797
1798        return ret;
1799}
1800
1801static int
1802group_sched_in(struct perf_event *group_event,
1803               struct perf_cpu_context *cpuctx,
1804               struct perf_event_context *ctx)
1805{
1806        struct perf_event *event, *partial_group = NULL;
1807        struct pmu *pmu = ctx->pmu;
1808        u64 now = ctx->time;
1809        bool simulate = false;
1810
1811        if (group_event->state == PERF_EVENT_STATE_OFF)
1812                return 0;
1813
1814        pmu->start_txn(pmu);
1815
1816        if (event_sched_in(group_event, cpuctx, ctx)) {
1817                pmu->cancel_txn(pmu);
1818                perf_cpu_hrtimer_restart(cpuctx);
1819                return -EAGAIN;
1820        }
1821
1822        /*
1823         * Schedule in siblings as one group (if any):
1824         */
1825        list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1826                if (event_sched_in(event, cpuctx, ctx)) {
1827                        partial_group = event;
1828                        goto group_error;
1829                }
1830        }
1831
1832        if (!pmu->commit_txn(pmu))
1833                return 0;
1834
1835group_error:
1836        /*
1837         * Groups can be scheduled in as one unit only, so undo any
1838         * partial group before returning:
1839         * The events up to the failed event are scheduled out normally,
1840         * tstamp_stopped will be updated.
1841         *
1842         * The failed events and the remaining siblings need to have
1843         * their timings updated as if they had gone thru event_sched_in()
1844         * and event_sched_out(). This is required to get consistent timings
1845         * across the group. This also takes care of the case where the group
1846         * could never be scheduled by ensuring tstamp_stopped is set to mark
1847         * the time the event was actually stopped, such that time delta
1848         * calculation in update_event_times() is correct.
1849         */
1850        list_for_each_entry(event, &group_event->sibling_list, group_entry) {
1851                if (event == partial_group)
1852                        simulate = true;
1853
1854                if (simulate) {
1855                        event->tstamp_running += now - event->tstamp_stopped;
1856                        event->tstamp_stopped = now;
1857                } else {
1858                        event_sched_out(event, cpuctx, ctx);
1859                }
1860        }
1861        event_sched_out(group_event, cpuctx, ctx);
1862
1863        pmu->cancel_txn(pmu);
1864
1865        perf_cpu_hrtimer_restart(cpuctx);
1866
1867        return -EAGAIN;
1868}
1869
1870/*
1871 * Work out whether we can put this event group on the CPU now.
1872 */
1873static int group_can_go_on(struct perf_event *event,
1874                           struct perf_cpu_context *cpuctx,
1875                           int can_add_hw)
1876{
1877        /*
1878         * Groups consisting entirely of software events can always go on.
1879         */
1880        if (event->group_flags & PERF_GROUP_SOFTWARE)
1881                return 1;
1882        /*
1883         * If an exclusive group is already on, no other hardware
1884         * events can go on.
1885         */
1886        if (cpuctx->exclusive)
1887                return 0;
1888        /*
1889         * If this group is exclusive and there are already
1890         * events on the CPU, it can't go on.
1891         */
1892        if (event->attr.exclusive && cpuctx->active_oncpu)
1893                return 0;
1894        /*
1895         * Otherwise, try to add it if all previous groups were able
1896         * to go on.
1897         */
1898        return can_add_hw;
1899}
1900
1901static void add_event_to_ctx(struct perf_event *event,
1902                               struct perf_event_context *ctx)
1903{
1904        u64 tstamp = perf_event_time(event);
1905
1906        list_add_event(event, ctx);
1907        perf_group_attach(event);
1908        event->tstamp_enabled = tstamp;
1909        event->tstamp_running = tstamp;
1910        event->tstamp_stopped = tstamp;
1911}
1912
1913static void task_ctx_sched_out(struct perf_event_context *ctx);
1914static void
1915ctx_sched_in(struct perf_event_context *ctx,
1916             struct perf_cpu_context *cpuctx,
1917             enum event_type_t event_type,
1918             struct task_struct *task);
1919
1920static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
1921                                struct perf_event_context *ctx,
1922                                struct task_struct *task)
1923{
1924        cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
1925        if (ctx)
1926                ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
1927        cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
1928        if (ctx)
1929                ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
1930}
1931
1932/*
1933 * Cross CPU call to install and enable a performance event
1934 *
1935 * Must be called with ctx->mutex held
1936 */
1937static int  __perf_install_in_context(void *info)
1938{
1939        struct perf_event *event = info;
1940        struct perf_event_context *ctx = event->ctx;
1941        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
1942        struct perf_event_context *task_ctx = cpuctx->task_ctx;
1943        struct task_struct *task = current;
1944
1945        perf_ctx_lock(cpuctx, task_ctx);
1946        perf_pmu_disable(cpuctx->ctx.pmu);
1947
1948        /*
1949         * If there was an active task_ctx schedule it out.
1950         */
1951        if (task_ctx)
1952                task_ctx_sched_out(task_ctx);
1953
1954        /*
1955         * If the context we're installing events in is not the
1956         * active task_ctx, flip them.
1957         */
1958        if (ctx->task && task_ctx != ctx) {
1959                if (task_ctx)
1960                        raw_spin_unlock(&task_ctx->lock);
1961                raw_spin_lock(&ctx->lock);
1962                task_ctx = ctx;
1963        }
1964
1965        if (task_ctx) {
1966                cpuctx->task_ctx = task_ctx;
1967                task = task_ctx->task;
1968        }
1969
1970        cpu_ctx_sched_out(cpuctx, EVENT_ALL);
1971
1972        update_context_time(ctx);
1973        /*
1974         * update cgrp time only if current cgrp
1975         * matches event->cgrp. Must be done before
1976         * calling add_event_to_ctx()
1977         */
1978        update_cgrp_time_from_event(event);
1979
1980        add_event_to_ctx(event, ctx);
1981
1982        /*
1983         * Schedule everything back in
1984         */
1985        perf_event_sched_in(cpuctx, task_ctx, task);
1986
1987        perf_pmu_enable(cpuctx->ctx.pmu);
1988        perf_ctx_unlock(cpuctx, task_ctx);
1989
1990        return 0;
1991}
1992
1993/*
1994 * Attach a performance event to a context
1995 *
1996 * First we add the event to the list with the hardware enable bit
1997 * in event->hw_config cleared.
1998 *
1999 * If the event is attached to a task which is on a CPU we use a smp
2000 * call to enable it in the task context. The task might have been
2001 * scheduled away, but we check this in the smp call again.
2002 */
2003static void
2004perf_install_in_context(struct perf_event_context *ctx,
2005                        struct perf_event *event,
2006                        int cpu)
2007{
2008        struct task_struct *task = ctx->task;
2009
2010        lockdep_assert_held(&ctx->mutex);
2011
2012        event->ctx = ctx;
2013        if (event->cpu != -1)
2014                event->cpu = cpu;
2015
2016        if (!task) {
2017                /*
2018                 * Per cpu events are installed via an smp call and
2019                 * the install is always successful.
2020                 */
2021                cpu_function_call(cpu, __perf_install_in_context, event);
2022                return;
2023        }
2024
2025retry:
2026        if (!task_function_call(task, __perf_install_in_context, event))
2027                return;
2028
2029        raw_spin_lock_irq(&ctx->lock);
2030        /*
2031         * If we failed to find a running task, but find the context active now
2032         * that we've acquired the ctx->lock, retry.
2033         */
2034        if (ctx->is_active) {
2035                raw_spin_unlock_irq(&ctx->lock);
2036                /*
2037                 * Reload the task pointer, it might have been changed by
2038                 * a concurrent perf_event_context_sched_out().
2039                 */
2040                task = ctx->task;
2041                goto retry;
2042        }
2043
2044        /*
2045         * Since the task isn't running, its safe to add the event, us holding
2046         * the ctx->lock ensures the task won't get scheduled in.
2047         */
2048        add_event_to_ctx(event, ctx);
2049        raw_spin_unlock_irq(&ctx->lock);
2050}
2051
2052/*
2053 * Put a event into inactive state and update time fields.
2054 * Enabling the leader of a group effectively enables all
2055 * the group members that aren't explicitly disabled, so we
2056 * have to update their ->tstamp_enabled also.
2057 * Note: this works for group members as well as group leaders
2058 * since the non-leader members' sibling_lists will be empty.
2059 */
2060static void __perf_event_mark_enabled(struct perf_event *event)
2061{
2062        struct perf_event *sub;
2063        u64 tstamp = perf_event_time(event);
2064
2065        event->state = PERF_EVENT_STATE_INACTIVE;
2066        event->tstamp_enabled = tstamp - event->total_time_enabled;
2067        list_for_each_entry(sub, &event->sibling_list, group_entry) {
2068                if (sub->state >= PERF_EVENT_STATE_INACTIVE)
2069                        sub->tstamp_enabled = tstamp - sub->total_time_enabled;
2070        }
2071}
2072
2073/*
2074 * Cross CPU call to enable a performance event
2075 */
2076static int __perf_event_enable(void *info)
2077{
2078        struct perf_event *event = info;
2079        struct perf_event_context *ctx = event->ctx;
2080        struct perf_event *leader = event->group_leader;
2081        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2082        int err;
2083
2084        /*
2085         * There's a time window between 'ctx->is_active' check
2086         * in perf_event_enable function and this place having:
2087         *   - IRQs on
2088         *   - ctx->lock unlocked
2089         *
2090         * where the task could be killed and 'ctx' deactivated
2091         * by perf_event_exit_task.
2092         */
2093        if (!ctx->is_active)
2094                return -EINVAL;
2095
2096        raw_spin_lock(&ctx->lock);
2097        update_context_time(ctx);
2098
2099        if (event->state >= PERF_EVENT_STATE_INACTIVE)
2100                goto unlock;
2101
2102        /*
2103         * set current task's cgroup time reference point
2104         */
2105        perf_cgroup_set_timestamp(current, ctx);
2106
2107        __perf_event_mark_enabled(event);
2108
2109        if (!event_filter_match(event)) {
2110                if (is_cgroup_event(event))
2111                        perf_cgroup_defer_enabled(event);
2112                goto unlock;
2113        }
2114
2115        /*
2116         * If the event is in a group and isn't the group leader,
2117         * then don't put it on unless the group is on.
2118         */
2119        if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE)
2120                goto unlock;
2121
2122        if (!group_can_go_on(event, cpuctx, 1)) {
2123                err = -EEXIST;
2124        } else {
2125                if (event == leader)
2126                        err = group_sched_in(event, cpuctx, ctx);
2127                else
2128                        err = event_sched_in(event, cpuctx, ctx);
2129        }
2130
2131        if (err) {
2132                /*
2133                 * If this event can't go on and it's part of a
2134                 * group, then the whole group has to come off.
2135                 */
2136                if (leader != event) {
2137                        group_sched_out(leader, cpuctx, ctx);
2138                        perf_cpu_hrtimer_restart(cpuctx);
2139                }
2140                if (leader->attr.pinned) {
2141                        update_group_times(leader);
2142                        leader->state = PERF_EVENT_STATE_ERROR;
2143                }
2144        }
2145
2146unlock:
2147        raw_spin_unlock(&ctx->lock);
2148
2149        return 0;
2150}
2151
2152/*
2153 * Enable a event.
2154 *
2155 * If event->ctx is a cloned context, callers must make sure that
2156 * every task struct that event->ctx->task could possibly point to
2157 * remains valid.  This condition is satisfied when called through
2158 * perf_event_for_each_child or perf_event_for_each as described
2159 * for perf_event_disable.
2160 */
2161void perf_event_enable(struct perf_event *event)
2162{
2163        struct perf_event_context *ctx = event->ctx;
2164        struct task_struct *task = ctx->task;
2165
2166        if (!task) {
2167                /*
2168                 * Enable the event on the cpu that it's on
2169                 */
2170                cpu_function_call(event->cpu, __perf_event_enable, event);
2171                return;
2172        }
2173
2174        raw_spin_lock_irq(&ctx->lock);
2175        if (event->state >= PERF_EVENT_STATE_INACTIVE)
2176                goto out;
2177
2178        /*
2179         * If the event is in error state, clear that first.
2180         * That way, if we see the event in error state below, we
2181         * know that it has gone back into error state, as distinct
2182         * from the task having been scheduled away before the
2183         * cross-call arrived.
2184         */
2185        if (event->state == PERF_EVENT_STATE_ERROR)
2186                event->state = PERF_EVENT_STATE_OFF;
2187
2188retry:
2189        if (!ctx->is_active) {
2190                __perf_event_mark_enabled(event);
2191                goto out;
2192        }
2193
2194        raw_spin_unlock_irq(&ctx->lock);
2195
2196        if (!task_function_call(task, __perf_event_enable, event))
2197                return;
2198
2199        raw_spin_lock_irq(&ctx->lock);
2200
2201        /*
2202         * If the context is active and the event is still off,
2203         * we need to retry the cross-call.
2204         */
2205        if (ctx->is_active && event->state == PERF_EVENT_STATE_OFF) {
2206                /*
2207                 * task could have been flipped by a concurrent
2208                 * perf_event_context_sched_out()
2209                 */
2210                task = ctx->task;
2211                goto retry;
2212        }
2213
2214out:
2215        raw_spin_unlock_irq(&ctx->lock);
2216}
2217EXPORT_SYMBOL_GPL(perf_event_enable);
2218
2219int perf_event_refresh(struct perf_event *event, int refresh)
2220{
2221        /*
2222         * not supported on inherited events
2223         */
2224        if (event->attr.inherit || !is_sampling_event(event))
2225                return -EINVAL;
2226
2227        atomic_add(refresh, &event->event_limit);
2228        perf_event_enable(event);
2229
2230        return 0;
2231}
2232EXPORT_SYMBOL_GPL(perf_event_refresh);
2233
2234static void ctx_sched_out(struct perf_event_context *ctx,
2235                          struct perf_cpu_context *cpuctx,
2236                          enum event_type_t event_type)
2237{
2238        struct perf_event *event;
2239        int is_active = ctx->is_active;
2240
2241        ctx->is_active &= ~event_type;
2242        if (likely(!ctx->nr_events))
2243                return;
2244
2245        update_context_time(ctx);
2246        update_cgrp_time_from_cpuctx(cpuctx);
2247        if (!ctx->nr_active)
2248                return;
2249
2250        perf_pmu_disable(ctx->pmu);
2251        if ((is_active & EVENT_PINNED) && (event_type & EVENT_PINNED)) {
2252                list_for_each_entry(event, &ctx->pinned_groups, group_entry)
2253                        group_sched_out(event, cpuctx, ctx);
2254        }
2255
2256        if ((is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE)) {
2257                list_for_each_entry(event, &ctx->flexible_groups, group_entry)
2258                        group_sched_out(event, cpuctx, ctx);
2259        }
2260        perf_pmu_enable(ctx->pmu);
2261}
2262
2263/*
2264 * Test whether two contexts are equivalent, i.e. whether they have both been
2265 * cloned from the same version of the same context.
2266 *
2267 * Equivalence is measured using a generation number in the context that is
2268 * incremented on each modification to it; see unclone_ctx(), list_add_event()
2269 * and list_del_event().
2270 */
2271static int context_equiv(struct perf_event_context *ctx1,
2272                         struct perf_event_context *ctx2)
2273{
2274        lockdep_assert_held(&ctx1->lock);
2275        lockdep_assert_held(&ctx2->lock);
2276
2277        /* Pinning disables the swap optimization */
2278        if (ctx1->pin_count || ctx2->pin_count)
2279                return 0;
2280
2281        /* If ctx1 is the parent of ctx2 */
2282        if (ctx1 == ctx2->parent_ctx && ctx1->generation == ctx2->parent_gen)
2283                return 1;
2284
2285        /* If ctx2 is the parent of ctx1 */
2286        if (ctx1->parent_ctx == ctx2 && ctx1->parent_gen == ctx2->generation)
2287                return 1;
2288
2289        /*
2290         * If ctx1 and ctx2 have the same parent; we flatten the parent
2291         * hierarchy, see perf_event_init_context().
2292         */
2293        if (ctx1->parent_ctx && ctx1->parent_ctx == ctx2->parent_ctx &&
2294                        ctx1->parent_gen == ctx2->parent_gen)
2295                return 1;
2296
2297        /* Unmatched */
2298        return 0;
2299}
2300
2301static void __perf_event_sync_stat(struct perf_event *event,
2302                                     struct perf_event *next_event)
2303{
2304        u64 value;
2305
2306        if (!event->attr.inherit_stat)
2307                return;
2308
2309        /*
2310         * Update the event value, we cannot use perf_event_read()
2311         * because we're in the middle of a context switch and have IRQs
2312         * disabled, which upsets smp_call_function_single(), however
2313         * we know the event must be on the current CPU, therefore we
2314         * don't need to use it.
2315         */
2316        switch (event->state) {
2317        case PERF_EVENT_STATE_ACTIVE:
2318                event->pmu->read(event);
2319                /* fall-through */
2320
2321        case PERF_EVENT_STATE_INACTIVE:
2322                update_event_times(event);
2323                break;
2324
2325        default:
2326                break;
2327        }
2328
2329        /*
2330         * In order to keep per-task stats reliable we need to flip the event
2331         * values when we flip the contexts.
2332         */
2333        value = local64_read(&next_event->count);
2334        value = local64_xchg(&event->count, value);
2335        local64_set(&next_event->count, value);
2336
2337        swap(event->total_time_enabled, next_event->total_time_enabled);
2338        swap(event->total_time_running, next_event->total_time_running);
2339
2340        /*
2341         * Since we swizzled the values, update the user visible data too.
2342         */
2343        perf_event_update_userpage(event);
2344        perf_event_update_userpage(next_event);
2345}
2346
2347static void perf_event_sync_stat(struct perf_event_context *ctx,
2348                                   struct perf_event_context *next_ctx)
2349{
2350        struct perf_event *event, *next_event;
2351
2352        if (!ctx->nr_stat)
2353                return;
2354
2355        update_context_time(ctx);
2356
2357        event = list_first_entry(&ctx->event_list,
2358                                   struct perf_event, event_entry);
2359
2360        next_event = list_first_entry(&next_ctx->event_list,
2361                                        struct perf_event, event_entry);
2362
2363        while (&event->event_entry != &ctx->event_list &&
2364               &next_event->event_entry != &next_ctx->event_list) {
2365
2366                __perf_event_sync_stat(event, next_event);
2367
2368                event = list_next_entry(event, event_entry);
2369                next_event = list_next_entry(next_event, event_entry);
2370        }
2371}
2372
2373static void perf_event_context_sched_out(struct task_struct *task, int ctxn,
2374                                         struct task_struct *next)
2375{
2376        struct perf_event_context *ctx = task->perf_event_ctxp[ctxn];
2377        struct perf_event_context *next_ctx;
2378        struct perf_event_context *parent, *next_parent;
2379        struct perf_cpu_context *cpuctx;
2380        int do_switch = 1;
2381
2382        if (likely(!ctx))
2383                return;
2384
2385        cpuctx = __get_cpu_context(ctx);
2386        if (!cpuctx->task_ctx)
2387                return;
2388
2389        rcu_read_lock();
2390        next_ctx = next->perf_event_ctxp[ctxn];
2391        if (!next_ctx)
2392                goto unlock;
2393
2394        parent = rcu_dereference(ctx->parent_ctx);
2395        next_parent = rcu_dereference(next_ctx->parent_ctx);
2396
2397        /* If neither context have a parent context; they cannot be clones. */
2398        if (!parent && !next_parent)
2399                goto unlock;
2400
2401        if (next_parent == ctx || next_ctx == parent || next_parent == parent) {
2402                /*
2403                 * Looks like the two contexts are clones, so we might be
2404                 * able to optimize the context switch.  We lock both
2405                 * contexts and check that they are clones under the
2406                 * lock (including re-checking that neither has been
2407                 * uncloned in the meantime).  It doesn't matter which
2408                 * order we take the locks because no other cpu could
2409                 * be trying to lock both of these tasks.
2410                 */
2411                raw_spin_lock(&ctx->lock);
2412                raw_spin_lock_nested(&next_ctx->lock, SINGLE_DEPTH_NESTING);
2413                if (context_equiv(ctx, next_ctx)) {
2414                        /*
2415                         * XXX do we need a memory barrier of sorts
2416                         * wrt to rcu_dereference() of perf_event_ctxp
2417                         */
2418                        task->perf_event_ctxp[ctxn] = next_ctx;
2419                        next->perf_event_ctxp[ctxn] = ctx;
2420                        ctx->task = next;
2421                        next_ctx->task = task;
2422                        do_switch = 0;
2423
2424                        perf_event_sync_stat(ctx, next_ctx);
2425                }
2426                raw_spin_unlock(&next_ctx->lock);
2427                raw_spin_unlock(&ctx->lock);
2428        }
2429unlock:
2430        rcu_read_unlock();
2431
2432        if (do_switch) {
2433                raw_spin_lock(&ctx->lock);
2434                ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2435                cpuctx->task_ctx = NULL;
2436                raw_spin_unlock(&ctx->lock);
2437        }
2438}
2439
2440#define for_each_task_context_nr(ctxn)                                  \
2441        for ((ctxn) = 0; (ctxn) < perf_nr_task_contexts; (ctxn)++)
2442
2443/*
2444 * Called from scheduler to remove the events of the current task,
2445 * with interrupts disabled.
2446 *
2447 * We stop each event and update the event value in event->count.
2448 *
2449 * This does not protect us against NMI, but disable()
2450 * sets the disabled bit in the control field of event _before_
2451 * accessing the event control register. If a NMI hits, then it will
2452 * not restart the event.
2453 */
2454void __perf_event_task_sched_out(struct task_struct *task,
2455                                 struct task_struct *next)
2456{
2457        int ctxn;
2458
2459        for_each_task_context_nr(ctxn)
2460                perf_event_context_sched_out(task, ctxn, next);
2461
2462        /*
2463         * if cgroup events exist on this CPU, then we need
2464         * to check if we have to switch out PMU state.
2465         * cgroup event are system-wide mode only
2466         */
2467        if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2468                perf_cgroup_sched_out(task, next);
2469}
2470
2471static void task_ctx_sched_out(struct perf_event_context *ctx)
2472{
2473        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
2474
2475        if (!cpuctx->task_ctx)
2476                return;
2477
2478        if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
2479                return;
2480
2481        ctx_sched_out(ctx, cpuctx, EVENT_ALL);
2482        cpuctx->task_ctx = NULL;
2483}
2484
2485/*
2486 * Called with IRQs disabled
2487 */
2488static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
2489                              enum event_type_t event_type)
2490{
2491        ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
2492}
2493
2494static void
2495ctx_pinned_sched_in(struct perf_event_context *ctx,
2496                    struct perf_cpu_context *cpuctx)
2497{
2498        struct perf_event *event;
2499
2500        list_for_each_entry(event, &ctx->pinned_groups, group_entry) {
2501                if (event->state <= PERF_EVENT_STATE_OFF)
2502                        continue;
2503                if (!event_filter_match(event))
2504                        continue;
2505
2506                /* may need to reset tstamp_enabled */
2507                if (is_cgroup_event(event))
2508                        perf_cgroup_mark_enabled(event, ctx);
2509
2510                if (group_can_go_on(event, cpuctx, 1))
2511                        group_sched_in(event, cpuctx, ctx);
2512
2513                /*
2514                 * If this pinned group hasn't been scheduled,
2515                 * put it in error state.
2516                 */
2517                if (event->state == PERF_EVENT_STATE_INACTIVE) {
2518                        update_group_times(event);
2519                        event->state = PERF_EVENT_STATE_ERROR;
2520                }
2521        }
2522}
2523
2524static void
2525ctx_flexible_sched_in(struct perf_event_context *ctx,
2526                      struct perf_cpu_context *cpuctx)
2527{
2528        struct perf_event *event;
2529        int can_add_hw = 1;
2530
2531        list_for_each_entry(event, &ctx->flexible_groups, group_entry) {
2532                /* Ignore events in OFF or ERROR state */
2533                if (event->state <= PERF_EVENT_STATE_OFF)
2534                        continue;
2535                /*
2536                 * Listen to the 'cpu' scheduling filter constraint
2537                 * of events:
2538                 */
2539                if (!event_filter_match(event))
2540                        continue;
2541
2542                /* may need to reset tstamp_enabled */
2543                if (is_cgroup_event(event))
2544                        perf_cgroup_mark_enabled(event, ctx);
2545
2546                if (group_can_go_on(event, cpuctx, can_add_hw)) {
2547                        if (group_sched_in(event, cpuctx, ctx))
2548                                can_add_hw = 0;
2549                }
2550        }
2551}
2552
2553static void
2554ctx_sched_in(struct perf_event_context *ctx,
2555             struct perf_cpu_context *cpuctx,
2556             enum event_type_t event_type,
2557             struct task_struct *task)
2558{
2559        u64 now;
2560        int is_active = ctx->is_active;
2561
2562        ctx->is_active |= event_type;
2563        if (likely(!ctx->nr_events))
2564                return;
2565
2566        now = perf_clock();
2567        ctx->timestamp = now;
2568        perf_cgroup_set_timestamp(task, ctx);
2569        /*
2570         * First go through the list and put on any pinned groups
2571         * in order to give them the best chance of going on.
2572         */
2573        if (!(is_active & EVENT_PINNED) && (event_type & EVENT_PINNED))
2574                ctx_pinned_sched_in(ctx, cpuctx);
2575
2576        /* Then walk through the lower prio flexible groups */
2577        if (!(is_active & EVENT_FLEXIBLE) && (event_type & EVENT_FLEXIBLE))
2578                ctx_flexible_sched_in(ctx, cpuctx);
2579}
2580
2581static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
2582                             enum event_type_t event_type,
2583                             struct task_struct *task)
2584{
2585        struct perf_event_context *ctx = &cpuctx->ctx;
2586
2587        ctx_sched_in(ctx, cpuctx, event_type, task);
2588}
2589
2590static void perf_event_context_sched_in(struct perf_event_context *ctx,
2591                                        struct task_struct *task)
2592{
2593        struct perf_cpu_context *cpuctx;
2594
2595        cpuctx = __get_cpu_context(ctx);
2596        if (cpuctx->task_ctx == ctx)
2597                return;
2598
2599        perf_ctx_lock(cpuctx, ctx);
2600        perf_pmu_disable(ctx->pmu);
2601        /*
2602         * We want to keep the following priority order:
2603         * cpu pinned (that don't need to move), task pinned,
2604         * cpu flexible, task flexible.
2605         */
2606        cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2607
2608        if (ctx->nr_events)
2609                cpuctx->task_ctx = ctx;
2610
2611        perf_event_sched_in(cpuctx, cpuctx->task_ctx, task);
2612
2613        perf_pmu_enable(ctx->pmu);
2614        perf_ctx_unlock(cpuctx, ctx);
2615
2616        /*
2617         * Since these rotations are per-cpu, we need to ensure the
2618         * cpu-context we got scheduled on is actually rotating.
2619         */
2620        perf_pmu_rotate_start(ctx->pmu);
2621}
2622
2623/*
2624 * When sampling the branck stack in system-wide, it may be necessary
2625 * to flush the stack on context switch. This happens when the branch
2626 * stack does not tag its entries with the pid of the current task.
2627 * Otherwise it becomes impossible to associate a branch entry with a
2628 * task. This ambiguity is more likely to appear when the branch stack
2629 * supports priv level filtering and the user sets it to monitor only
2630 * at the user level (which could be a useful measurement in system-wide
2631 * mode). In that case, the risk is high of having a branch stack with
2632 * branch from multiple tasks. Flushing may mean dropping the existing
2633 * entries or stashing them somewhere in the PMU specific code layer.
2634 *
2635 * This function provides the context switch callback to the lower code
2636 * layer. It is invoked ONLY when there is at least one system-wide context
2637 * with at least one active event using taken branch sampling.
2638 */
2639static void perf_branch_stack_sched_in(struct task_struct *prev,
2640                                       struct task_struct *task)
2641{
2642        struct perf_cpu_context *cpuctx;
2643        struct pmu *pmu;
2644        unsigned long flags;
2645
2646        /* no need to flush branch stack if not changing task */
2647        if (prev == task)
2648                return;
2649
2650        local_irq_save(flags);
2651
2652        rcu_read_lock();
2653
2654        list_for_each_entry_rcu(pmu, &pmus, entry) {
2655                cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
2656
2657                /*
2658                 * check if the context has at least one
2659                 * event using PERF_SAMPLE_BRANCH_STACK
2660                 */
2661                if (cpuctx->ctx.nr_branch_stack > 0
2662                    && pmu->flush_branch_stack) {
2663
2664                        perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2665
2666                        perf_pmu_disable(pmu);
2667
2668                        pmu->flush_branch_stack();
2669
2670                        perf_pmu_enable(pmu);
2671
2672                        perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2673                }
2674        }
2675
2676        rcu_read_unlock();
2677
2678        local_irq_restore(flags);
2679}
2680
2681/*
2682 * Called from scheduler to add the events of the current task
2683 * with interrupts disabled.
2684 *
2685 * We restore the event value and then enable it.
2686 *
2687 * This does not protect us against NMI, but enable()
2688 * sets the enabled bit in the control field of event _before_
2689 * accessing the event control register. If a NMI hits, then it will
2690 * keep the event running.
2691 */
2692void __perf_event_task_sched_in(struct task_struct *prev,
2693                                struct task_struct *task)
2694{
2695        struct perf_event_context *ctx;
2696        int ctxn;
2697
2698        for_each_task_context_nr(ctxn) {
2699                ctx = task->perf_event_ctxp[ctxn];
2700                if (likely(!ctx))
2701                        continue;
2702
2703                perf_event_context_sched_in(ctx, task);
2704        }
2705        /*
2706         * if cgroup events exist on this CPU, then we need
2707         * to check if we have to switch in PMU state.
2708         * cgroup event are system-wide mode only
2709         */
2710        if (atomic_read(this_cpu_ptr(&perf_cgroup_events)))
2711                perf_cgroup_sched_in(prev, task);
2712
2713        /* check for system-wide branch_stack events */
2714        if (atomic_read(this_cpu_ptr(&perf_branch_stack_events)))
2715                perf_branch_stack_sched_in(prev, task);
2716}
2717
2718static u64 perf_calculate_period(struct perf_event *event, u64 nsec, u64 count)
2719{
2720        u64 frequency = event->attr.sample_freq;
2721        u64 sec = NSEC_PER_SEC;
2722        u64 divisor, dividend;
2723
2724        int count_fls, nsec_fls, frequency_fls, sec_fls;
2725
2726        count_fls = fls64(count);
2727        nsec_fls = fls64(nsec);
2728        frequency_fls = fls64(frequency);
2729        sec_fls = 30;
2730
2731        /*
2732         * We got @count in @nsec, with a target of sample_freq HZ
2733         * the target period becomes:
2734         *
2735         *             @count * 10^9
2736         * period = -------------------
2737         *          @nsec * sample_freq
2738         *
2739         */
2740
2741        /*
2742         * Reduce accuracy by one bit such that @a and @b converge
2743         * to a similar magnitude.
2744         */
2745#define REDUCE_FLS(a, b)                \
2746do {                                    \
2747        if (a##_fls > b##_fls) {        \
2748                a >>= 1;                \
2749                a##_fls--;              \
2750        } else {                        \
2751                b >>= 1;                \
2752                b##_fls--;              \
2753        }                               \
2754} while (0)
2755
2756        /*
2757         * Reduce accuracy until either term fits in a u64, then proceed with
2758         * the other, so that finally we can do a u64/u64 division.
2759         */
2760        while (count_fls + sec_fls > 64 && nsec_fls + frequency_fls > 64) {
2761                REDUCE_FLS(nsec, frequency);
2762                REDUCE_FLS(sec, count);
2763        }
2764
2765        if (count_fls + sec_fls > 64) {
2766                divisor = nsec * frequency;
2767
2768                while (count_fls + sec_fls > 64) {
2769                        REDUCE_FLS(count, sec);
2770                        divisor >>= 1;
2771                }
2772
2773                dividend = count * sec;
2774        } else {
2775                dividend = count * sec;
2776
2777                while (nsec_fls + frequency_fls > 64) {
2778                        REDUCE_FLS(nsec, frequency);
2779                        dividend >>= 1;
2780                }
2781
2782                divisor = nsec * frequency;
2783        }
2784
2785        if (!divisor)
2786                return dividend;
2787
2788        return div64_u64(dividend, divisor);
2789}
2790
2791static DEFINE_PER_CPU(int, perf_throttled_count);
2792static DEFINE_PER_CPU(u64, perf_throttled_seq);
2793
2794static void perf_adjust_period(struct perf_event *event, u64 nsec, u64 count, bool disable)
2795{
2796        struct hw_perf_event *hwc = &event->hw;
2797        s64 period, sample_period;
2798        s64 delta;
2799
2800        period = perf_calculate_period(event, nsec, count);
2801
2802        delta = (s64)(period - hwc->sample_period);
2803        delta = (delta + 7) / 8; /* low pass filter */
2804
2805        sample_period = hwc->sample_period + delta;
2806
2807        if (!sample_period)
2808                sample_period = 1;
2809
2810        hwc->sample_period = sample_period;
2811
2812        if (local64_read(&hwc->period_left) > 8*sample_period) {
2813                if (disable)
2814                        event->pmu->stop(event, PERF_EF_UPDATE);
2815
2816                local64_set(&hwc->period_left, 0);
2817
2818                if (disable)
2819                        event->pmu->start(event, PERF_EF_RELOAD);
2820        }
2821}
2822
2823/*
2824 * combine freq adjustment with unthrottling to avoid two passes over the
2825 * events. At the same time, make sure, having freq events does not change
2826 * the rate of unthrottling as that would introduce bias.
2827 */
2828static void perf_adjust_freq_unthr_context(struct perf_event_context *ctx,
2829                                           int needs_unthr)
2830{
2831        struct perf_event *event;
2832        struct hw_perf_event *hwc;
2833        u64 now, period = TICK_NSEC;
2834        s64 delta;
2835
2836        /*
2837         * only need to iterate over all events iff:
2838         * - context have events in frequency mode (needs freq adjust)
2839         * - there are events to unthrottle on this cpu
2840         */
2841        if (!(ctx->nr_freq || needs_unthr))
2842                return;
2843
2844        raw_spin_lock(&ctx->lock);
2845        perf_pmu_disable(ctx->pmu);
2846
2847        list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
2848                if (event->state != PERF_EVENT_STATE_ACTIVE)
2849                        continue;
2850
2851                if (!event_filter_match(event))
2852                        continue;
2853
2854                perf_pmu_disable(event->pmu);
2855
2856                hwc = &event->hw;
2857
2858                if (hwc->interrupts == MAX_INTERRUPTS) {
2859                        hwc->interrupts = 0;
2860                        perf_log_throttle(event, 1);
2861                        event->pmu->start(event, 0);
2862                }
2863
2864                if (!event->attr.freq || !event->attr.sample_freq)
2865                        goto next;
2866
2867                /*
2868                 * stop the event and update event->count
2869                 */
2870                event->pmu->stop(event, PERF_EF_UPDATE);
2871
2872                now = local64_read(&event->count);
2873                delta = now - hwc->freq_count_stamp;
2874                hwc->freq_count_stamp = now;
2875
2876                /*
2877                 * restart the event
2878                 * reload only if value has changed
2879                 * we have stopped the event so tell that
2880                 * to perf_adjust_period() to avoid stopping it
2881                 * twice.
2882                 */
2883                if (delta > 0)
2884                        perf_adjust_period(event, period, delta, false);
2885
2886                event->pmu->start(event, delta > 0 ? PERF_EF_RELOAD : 0);
2887        next:
2888                perf_pmu_enable(event->pmu);
2889        }
2890
2891        perf_pmu_enable(ctx->pmu);
2892        raw_spin_unlock(&ctx->lock);
2893}
2894
2895/*
2896 * Round-robin a context's events:
2897 */
2898static void rotate_ctx(struct perf_event_context *ctx)
2899{
2900        /*
2901         * Rotate the first entry last of non-pinned groups. Rotation might be
2902         * disabled by the inheritance code.
2903         */
2904        if (!ctx->rotate_disable)
2905                list_rotate_left(&ctx->flexible_groups);
2906}
2907
2908/*
2909 * perf_pmu_rotate_start() and perf_rotate_context() are fully serialized
2910 * because they're strictly cpu affine and rotate_start is called with IRQs
2911 * disabled, while rotate_context is called from IRQ context.
2912 */
2913static int perf_rotate_context(struct perf_cpu_context *cpuctx)
2914{
2915        struct perf_event_context *ctx = NULL;
2916        int rotate = 0, remove = 1;
2917
2918        if (cpuctx->ctx.nr_events) {
2919                remove = 0;
2920                if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
2921                        rotate = 1;
2922        }
2923
2924        ctx = cpuctx->task_ctx;
2925        if (ctx && ctx->nr_events) {
2926                remove = 0;
2927                if (ctx->nr_events != ctx->nr_active)
2928                        rotate = 1;
2929        }
2930
2931        if (!rotate)
2932                goto done;
2933
2934        perf_ctx_lock(cpuctx, cpuctx->task_ctx);
2935        perf_pmu_disable(cpuctx->ctx.pmu);
2936
2937        cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
2938        if (ctx)
2939                ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
2940
2941        rotate_ctx(&cpuctx->ctx);
2942        if (ctx)
2943                rotate_ctx(ctx);
2944
2945        perf_event_sched_in(cpuctx, ctx, current);
2946
2947        perf_pmu_enable(cpuctx->ctx.pmu);
2948        perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
2949done:
2950        if (remove)
2951                list_del_init(&cpuctx->rotation_list);
2952
2953        return rotate;
2954}
2955
2956#ifdef CONFIG_NO_HZ_FULL
2957bool perf_event_can_stop_tick(void)
2958{
2959        if (atomic_read(&nr_freq_events) ||
2960            __this_cpu_read(perf_throttled_count))
2961                return false;
2962        else
2963                return true;
2964}
2965#endif
2966
2967void perf_event_task_tick(void)
2968{
2969        struct list_head *head = this_cpu_ptr(&rotation_list);
2970        struct perf_cpu_context *cpuctx, *tmp;
2971        struct perf_event_context *ctx;
2972        int throttled;
2973
2974        WARN_ON(!irqs_disabled());
2975
2976        __this_cpu_inc(perf_throttled_seq);
2977        throttled = __this_cpu_xchg(perf_throttled_count, 0);
2978
2979        list_for_each_entry_safe(cpuctx, tmp, head, rotation_list) {
2980                ctx = &cpuctx->ctx;
2981                perf_adjust_freq_unthr_context(ctx, throttled);
2982
2983                ctx = cpuctx->task_ctx;
2984                if (ctx)
2985                        perf_adjust_freq_unthr_context(ctx, throttled);
2986        }
2987}
2988
2989static int event_enable_on_exec(struct perf_event *event,
2990                                struct perf_event_context *ctx)
2991{
2992        if (!event->attr.enable_on_exec)
2993                return 0;
2994
2995        event->attr.enable_on_exec = 0;
2996        if (event->state >= PERF_EVENT_STATE_INACTIVE)
2997                return 0;
2998
2999        __perf_event_mark_enabled(event);
3000
3001        return 1;
3002}
3003
3004/*
3005 * Enable all of a task's events that have been marked enable-on-exec.
3006 * This expects task == current.
3007 */
3008static void perf_event_enable_on_exec(struct perf_event_context *ctx)
3009{
3010        struct perf_event_context *clone_ctx = NULL;
3011        struct perf_event *event;
3012        unsigned long flags;
3013        int enabled = 0;
3014        int ret;
3015
3016        local_irq_save(flags);
3017        if (!ctx || !ctx->nr_events)
3018                goto out;
3019
3020        /*
3021         * We must ctxsw out cgroup events to avoid conflict
3022         * when invoking perf_task_event_sched_in() later on
3023         * in this function. Otherwise we end up trying to
3024         * ctxswin cgroup events which are already scheduled
3025         * in.
3026         */
3027        perf_cgroup_sched_out(current, NULL);
3028
3029        raw_spin_lock(&ctx->lock);
3030        task_ctx_sched_out(ctx);
3031
3032        list_for_each_entry(event, &ctx->event_list, event_entry) {
3033                ret = event_enable_on_exec(event, ctx);
3034                if (ret)
3035                        enabled = 1;
3036        }
3037
3038        /*
3039         * Unclone this context if we enabled any event.
3040         */
3041        if (enabled)
3042                clone_ctx = unclone_ctx(ctx);
3043
3044        raw_spin_unlock(&ctx->lock);
3045
3046        /*
3047         * Also calls ctxswin for cgroup events, if any:
3048         */
3049        perf_event_context_sched_in(ctx, ctx->task);
3050out:
3051        local_irq_restore(flags);
3052
3053        if (clone_ctx)
3054                put_ctx(clone_ctx);
3055}
3056
3057void perf_event_exec(void)
3058{
3059        struct perf_event_context *ctx;
3060        int ctxn;
3061
3062        rcu_read_lock();
3063        for_each_task_context_nr(ctxn) {
3064                ctx = current->perf_event_ctxp[ctxn];
3065                if (!ctx)
3066                        continue;
3067
3068                perf_event_enable_on_exec(ctx);
3069        }
3070        rcu_read_unlock();
3071}
3072
3073/*
3074 * Cross CPU call to read the hardware event
3075 */
3076static void __perf_event_read(void *info)
3077{
3078        struct perf_event *event = info;
3079        struct perf_event_context *ctx = event->ctx;
3080        struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
3081
3082        /*
3083         * If this is a task context, we need to check whether it is
3084         * the current task context of this cpu.  If not it has been
3085         * scheduled out before the smp call arrived.  In that case
3086         * event->count would have been updated to a recent sample
3087         * when the event was scheduled out.
3088         */
3089        if (ctx->task && cpuctx->task_ctx != ctx)
3090                return;
3091
3092        raw_spin_lock(&ctx->lock);
3093        if (ctx->is_active) {
3094                update_context_time(ctx);
3095                update_cgrp_time_from_event(event);
3096        }
3097        update_event_times(event);
3098        if (event->state == PERF_EVENT_STATE_ACTIVE)
3099                event->pmu->read(event);
3100        raw_spin_unlock(&ctx->lock);
3101}
3102
3103static inline u64 perf_event_count(struct perf_event *event)
3104{
3105        return local64_read(&event->count) + atomic64_read(&event->child_count);
3106}
3107
3108static u64 perf_event_read(struct perf_event *event)
3109{
3110        /*
3111         * If event is enabled and currently active on a CPU, update the
3112         * value in the event structure:
3113         */
3114        if (event->state == PERF_EVENT_STATE_ACTIVE) {
3115                smp_call_function_single(event->oncpu,
3116                                         __perf_event_read, event, 1);
3117        } else if (event->state == PERF_EVENT_STATE_INACTIVE) {
3118                struct perf_event_context *ctx = event->ctx;
3119                unsigned long flags;
3120
3121                raw_spin_lock_irqsave(&ctx->lock, flags);
3122                /*
3123                 * may read while context is not active
3124                 * (e.g., thread is blocked), in that case
3125                 * we cannot update context time
3126                 */
3127                if (ctx->is_active) {
3128                        update_context_time(ctx);
3129                        update_cgrp_time_from_event(event);
3130                }
3131                update_event_times(event);
3132                raw_spin_unlock_irqrestore(&ctx->lock, flags);
3133        }
3134
3135        return perf_event_count(event);
3136}
3137
3138/*
3139 * Initialize the perf_event context in a task_struct:
3140 */
3141static void __perf_event_init_context(struct perf_event_context *ctx)
3142{
3143        raw_spin_lock_init(&ctx->lock);
3144        mutex_init(&ctx->mutex);
3145        INIT_LIST_HEAD(&ctx->pinned_groups);
3146        INIT_LIST_HEAD(&ctx->flexible_groups);
3147        INIT_LIST_HEAD(&ctx->event_list);
3148        atomic_set(&ctx->refcount, 1);
3149        INIT_DELAYED_WORK(&ctx->orphans_remove, orphans_remove_work);
3150}
3151
3152static struct perf_event_context *
3153alloc_perf_context(struct pmu *pmu, struct task_struct *task)
3154{
3155        struct perf_event_context *ctx;
3156
3157        ctx = kzalloc(sizeof(struct perf_event_context), GFP_KERNEL);
3158        if (!ctx)
3159                return NULL;
3160
3161        __perf_event_init_context(ctx);
3162        if (task) {
3163                ctx->task = task;
3164                get_task_struct(task);
3165        }
3166        ctx->pmu = pmu;
3167
3168        return ctx;
3169}
3170
3171static struct task_struct *
3172find_lively_task_by_vpid(pid_t vpid)
3173{
3174        struct task_struct *task;
3175        int err;
3176
3177        rcu_read_lock();
3178        if (!vpid)
3179                task = current;
3180        else
3181                task = find_task_by_vpid(vpid);
3182        if (task)
3183                get_task_struct(task);
3184        rcu_read_unlock();
3185
3186        if (!task)
3187                return ERR_PTR(-ESRCH);
3188
3189        /* Reuse ptrace permission checks for now. */
3190        err = -EACCES;
3191        if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS))
3192                goto errout;
3193
3194        return task;
3195errout:
3196        put_task_struct(task);
3197        return ERR_PTR(err);
3198
3199}
3200
3201/*
3202 * Returns a matching context with refcount and pincount.
3203 */
3204static struct perf_event_context *
3205find_get_context(struct pmu *pmu, struct task_struct *task, int cpu)
3206{
3207        struct perf_event_context *ctx, *clone_ctx = NULL;
3208        struct perf_cpu_context *cpuctx;
3209        unsigned long flags;
3210        int ctxn, err;
3211
3212        if (!task) {
3213                /* Must be root to operate on a CPU event: */
3214                if (perf_paranoid_cpu() && !capable(CAP_SYS_ADMIN))
3215                        return ERR_PTR(-EACCES);
3216
3217                /*
3218                 * We could be clever and allow to attach a event to an
3219                 * offline CPU and activate it when the CPU comes up, but
3220                 * that's for later.
3221                 */
3222                if (!cpu_online(cpu))
3223                        return ERR_PTR(-ENODEV);
3224
3225                cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
3226                ctx = &cpuctx->ctx;
3227                get_ctx(ctx);
3228                ++ctx->pin_count;
3229
3230                return ctx;
3231        }
3232
3233        err = -EINVAL;
3234        ctxn = pmu->task_ctx_nr;
3235        if (ctxn < 0)
3236                goto errout;
3237
3238retry:
3239        ctx = perf_lock_task_context(task, ctxn, &flags);
3240        if (ctx) {
3241                clone_ctx = unclone_ctx(ctx);
3242                ++ctx->pin_count;
3243                raw_spin_unlock_irqrestore(&ctx->lock, flags);
3244
3245                if (clone_ctx)
3246                        put_ctx(clone_ctx);
3247        } else {
3248                ctx = alloc_perf_context(pmu, task);
3249                err = -ENOMEM;
3250                if (!ctx)
3251                        goto errout;
3252
3253                err = 0;
3254                mutex_lock(&task->perf_event_mutex);
3255                /*
3256                 * If it has already passed perf_event_exit_task().
3257                 * we must see PF_EXITING, it takes this mutex too.
3258                 */
3259                if (task->flags & PF_EXITING)
3260                        err = -ESRCH;
3261                else if (task->perf_event_ctxp[ctxn])
3262                        err = -EAGAIN;
3263                else {
3264                        get_ctx(ctx);
3265                        ++ctx->pin_count;
3266                        rcu_assign_pointer(task->perf_event_ctxp[ctxn], ctx);
3267                }
3268                mutex_unlock(&task->perf_event_mutex);
3269
3270                if (unlikely(err)) {
3271                        put_ctx(ctx);
3272
3273                        if (err == -EAGAIN)
3274                                goto retry;
3275                        goto errout;
3276                }
3277        }
3278
3279        return ctx;
3280
3281errout:
3282        return ERR_PTR(err);
3283}
3284
3285static void perf_event_free_filter(struct perf_event *event);
3286
3287static void free_event_rcu(struct rcu_head *head)
3288{
3289        struct perf_event *event;
3290
3291        event = container_of(head, struct perf_event, rcu_head);
3292        if (event->ns)
3293                put_pid_ns(event->ns);
3294        perf_event_free_filter(event);
3295        kfree(event);
3296}
3297
3298static void ring_buffer_put(struct ring_buffer *rb);
3299static void ring_buffer_attach(struct perf_event *event,
3300                               struct ring_buffer *rb);
3301
3302static void unaccount_event_cpu(struct perf_event *event, int cpu)
3303{
3304        if (event->parent)
3305                return;
3306
3307        if (has_branch_stack(event)) {
3308                if (!(event->attach_state & PERF_ATTACH_TASK))
3309                        atomic_dec(&per_cpu(perf_branch_stack_events, cpu));
3310        }
3311        if (is_cgroup_event(event))
3312                atomic_dec(&per_cpu(perf_cgroup_events, cpu));
3313}
3314
3315static void unaccount_event(struct perf_event *event)
3316{
3317        if (event->parent)
3318                return;
3319
3320        if (event->attach_state & PERF_ATTACH_TASK)
3321                static_key_slow_dec_deferred(&perf_sched_events);
3322        if (event->attr.mmap || event->attr.mmap_data)
3323                atomic_dec(&nr_mmap_events);
3324        if (event->attr.comm)
3325                atomic_dec(&nr_comm_events);
3326        if (event->attr.task)
3327                atomic_dec(&nr_task_events);
3328        if (event->attr.freq)
3329                atomic_dec(&nr_freq_events);
3330        if (is_cgroup_event(event))
3331                static_key_slow_dec_deferred(&perf_sched_events);
3332        if (has_branch_stack(event))
3333                static_key_slow_dec_deferred(&perf_sched_events);
3334
3335        unaccount_event_cpu(event, event->cpu);
3336}
3337
3338static void __free_event(struct perf_event *event)
3339{
3340        if (!event->parent) {
3341                if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN)
3342                        put_callchain_buffers();
3343        }
3344
3345        if (event->destroy)
3346                event->destroy(event);
3347
3348        if (event->ctx)
3349                put_ctx(event->ctx);
3350
3351        if (event->pmu)
3352                module_put(event->pmu->module);
3353
3354        call_rcu(&event->rcu_head, free_event_rcu);
3355}
3356
3357static void _free_event(struct perf_event *event)
3358{
3359        irq_work_sync(&event->pending);
3360
3361        unaccount_event(event);
3362
3363        if (event->rb) {
3364                /*
3365                 * Can happen when we close an event with re-directed output.
3366                 *
3367                 * Since we have a 0 refcount, perf_mmap_close() will skip
3368                 * over us; possibly making our ring_buffer_put() the last.
3369                 */
3370                mutex_lock(&event->mmap_mutex);
3371                ring_buffer_attach(event, NULL);
3372                mutex_unlock(&event->mmap_mutex);
3373        }
3374
3375        if (is_cgroup_event(event))
3376                perf_detach_cgroup(event);
3377
3378        __free_event(event);
3379}
3380
3381/*
3382 * Used to free events which have a known refcount of 1, such as in error paths
3383 * where the event isn't exposed yet and inherited events.
3384 */
3385static void free_event(struct perf_event *event)
3386{
3387        if (WARN(atomic_long_cmpxchg(&event->refcount, 1, 0) != 1,
3388                                "unexpected event refcount: %ld; ptr=%p\n",
3389                                atomic_long_read(&event->refcount), event)) {
3390                /* leak to avoid use-after-free */
3391                return;
3392        }
3393
3394        _free_event(event);
3395}
3396
3397/*
3398 * Remove user event from the owner task.
3399 */
3400static void perf_remove_from_owner(struct perf_event *event)
3401{
3402        struct task_struct *owner;
3403
3404        rcu_read_lock();
3405        owner = ACCESS_ONCE(event->owner);
3406        /*
3407         * Matches the smp_wmb() in perf_event_exit_task(). If we observe
3408         * !owner it means the list deletion is complete and we can indeed
3409         * free this event, otherwise we need to serialize on
3410         * owner->perf_event_mutex.
3411         */
3412        smp_read_barrier_depends();
3413        if (owner) {
3414                /*
3415                 * Since delayed_put_task_struct() also drops the last
3416                 * task reference we can safely take a new reference
3417                 * while holding the rcu_read_lock().
3418                 */
3419                get_task_struct(owner);
3420        }
3421        rcu_read_unlock();
3422
3423        if (owner) {
3424                mutex_lock(&owner->perf_event_mutex);
3425                /*
3426                 * We have to re-check the event->owner field, if it is cleared
3427                 * we raced with perf_event_exit_task(), acquiring the mutex
3428                 * ensured they're done, and we can proceed with freeing the
3429                 * event.
3430                 */
3431                if (event->owner)
3432                        list_del_init(&event->owner_entry);
3433                mutex_unlock(&owner->perf_event_mutex);
3434                put_task_struct(owner);
3435        }
3436}
3437
3438/*
3439 * Called when the last reference to the file is gone.
3440 */
3441static void put_event(struct perf_event *event)
3442{
3443        struct perf_event_context *ctx = event->ctx;
3444
3445        if (!atomic_long_dec_and_test(&event->refcount))
3446                return;
3447
3448        if (!is_kernel_event(event))
3449                perf_remove_from_owner(event);
3450
3451        WARN_ON_ONCE(ctx->parent_ctx);
3452        /*
3453         * There are two ways this annotation is useful:
3454         *
3455         *  1) there is a lock recursion from perf_event_exit_task
3456         *     see the comment there.
3457         *
3458         *  2) there is a lock-inversion with mmap_sem through
3459         *     perf_event_read_group(), which takes faults while
3460         *     holding ctx->mutex, however this is called after
3461         *     the last filedesc died, so there is no possibility
3462         *     to trigger the AB-BA case.
3463         */
3464        mutex_lock_nested(&ctx->mutex, SINGLE_DEPTH_NESTING);
3465        perf_remove_from_context(event, true);
3466        mutex_unlock(&ctx->mutex);
3467
3468        _free_event(event);
3469}
3470
3471int perf_event_release_kernel(struct perf_event *event)
3472{
3473        put_event(event);
3474        return 0;
3475}
3476EXPORT_SYMBOL_GPL(perf_event_release_kernel);
3477
3478static int perf_release(struct inode *inode, struct file *file)
3479{
3480        put_event(file->private_data);
3481        return 0;
3482}
3483
3484/*
3485 * Remove all orphanes events from the context.
3486 */
3487static void orphans_remove_work(struct work_struct *work)
3488{
3489        struct perf_event_context *ctx;
3490        struct perf_event *event, *tmp;
3491
3492        ctx = container_of(work, struct perf_event_context,
3493                           orphans_remove.work);
3494
3495        mutex_lock(&ctx->mutex);
3496        list_for_each_entry_safe(event, tmp, &ctx->event_list, event_entry) {
3497                struct perf_event *parent_event = event->parent;
3498
3499                if (!is_orphaned_child(event))
3500                        continue;
3501
3502                perf_remove_from_context(event, true);
3503
3504                mutex_lock(&parent_event->child_mutex);
3505                list_del_init(&event->child_list);
3506                mutex_unlock(&parent_event->child_mutex);
3507
3508                free_event(event);
3509                put_event(parent_event);
3510        }
3511
3512        raw_spin_lock_irq(&ctx->lock);
3513        ctx->orphans_remove_sched = false;
3514        raw_spin_unlock_irq(&ctx->lock);
3515        mutex_unlock(&ctx->mutex);
3516
3517        put_ctx(ctx);
3518}
3519
3520u64 perf_event_read_value(struct perf_event *event, u64 *enabled, u64 *running)
3521{
3522        struct perf_event *child;
3523        u64 total = 0;
3524
3525        *enabled = 0;
3526        *running = 0;
3527
3528        mutex_lock(&event->child_mutex);
3529        total += perf_event_read(event);
3530        *enabled += event->total_time_enabled +
3531                        atomic64_read(&event->child_total_time_enabled);
3532        *running += event->total_time_running +
3533                        atomic64_read(&event->child_total_time_running);
3534
3535        list_for_each_entry(child, &event->child_list, child_list) {
3536                total += perf_event_read(child);
3537                *enabled += child->total_time_enabled;
3538                *running += child->total_time_running;
3539        }
3540        mutex_unlock(&event->child_mutex);
3541
3542        return total;
3543}
3544EXPORT_SYMBOL_GPL(perf_event_read_value);
3545
3546static int perf_event_read_group(struct perf_event *event,
3547                                   u64 read_format, char __user *buf)
3548{
3549        struct perf_event *leader = event->group_leader, *sub;
3550        int n = 0, size = 0, ret = -EFAULT;
3551        struct perf_event_context *ctx = leader->ctx;
3552        u64 values[5];
3553        u64 count, enabled, running;
3554
3555        mutex_lock(&ctx->mutex);
3556        count = perf_event_read_value(leader, &enabled, &running);
3557
3558        values[n++] = 1 + leader->nr_siblings;
3559        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3560                values[n++] = enabled;
3561        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3562                values[n++] = running;
3563        values[n++] = count;
3564        if (read_format & PERF_FORMAT_ID)
3565                values[n++] = primary_event_id(leader);
3566
3567        size = n * sizeof(u64);
3568
3569        if (copy_to_user(buf, values, size))
3570                goto unlock;
3571
3572        ret = size;
3573
3574        list_for_each_entry(sub, &leader->sibling_list, group_entry) {
3575                n = 0;
3576
3577                values[n++] = perf_event_read_value(sub, &enabled, &running);
3578                if (read_format & PERF_FORMAT_ID)
3579                        values[n++] = primary_event_id(sub);
3580
3581                size = n * sizeof(u64);
3582
3583                if (copy_to_user(buf + ret, values, size)) {
3584                        ret = -EFAULT;
3585                        goto unlock;
3586                }
3587
3588                ret += size;
3589        }
3590unlock:
3591        mutex_unlock(&ctx->mutex);
3592
3593        return ret;
3594}
3595
3596static int perf_event_read_one(struct perf_event *event,
3597                                 u64 read_format, char __user *buf)
3598{
3599        u64 enabled, running;
3600        u64 values[4];
3601        int n = 0;
3602
3603        values[n++] = perf_event_read_value(event, &enabled, &running);
3604        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
3605                values[n++] = enabled;
3606        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
3607                values[n++] = running;
3608        if (read_format & PERF_FORMAT_ID)
3609                values[n++] = primary_event_id(event);
3610
3611        if (copy_to_user(buf, values, n * sizeof(u64)))
3612                return -EFAULT;
3613
3614        return n * sizeof(u64);
3615}
3616
3617static bool is_event_hup(struct perf_event *event)
3618{
3619        bool no_children;
3620
3621        if (event->state != PERF_EVENT_STATE_EXIT)
3622                return false;
3623
3624        mutex_lock(&event->child_mutex);
3625        no_children = list_empty(&event->child_list);
3626        mutex_unlock(&event->child_mutex);
3627        return no_children;
3628}
3629
3630/*
3631 * Read the performance event - simple non blocking version for now
3632 */
3633static ssize_t
3634perf_read_hw(struct perf_event *event, char __user *buf, size_t count)
3635{
3636        u64 read_format = event->attr.read_format;
3637        int ret;
3638
3639        /*
3640         * Return end-of-file for a read on a event that is in
3641         * error state (i.e. because it was pinned but it couldn't be
3642         * scheduled on to the CPU at some point).
3643         */
3644        if (event->state == PERF_EVENT_STATE_ERROR)
3645                return 0;
3646
3647        if (count < event->read_size)
3648                return -ENOSPC;
3649
3650        WARN_ON_ONCE(event->ctx->parent_ctx);
3651        if (read_format & PERF_FORMAT_GROUP)
3652                ret = perf_event_read_group(event, read_format, buf);
3653        else
3654                ret = perf_event_read_one(event, read_format, buf);
3655
3656        return ret;
3657}
3658
3659static ssize_t
3660perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
3661{
3662        struct perf_event *event = file->private_data;
3663
3664        return perf_read_hw(event, buf, count);
3665}
3666
3667static unsigned int perf_poll(struct file *file, poll_table *wait)
3668{
3669        struct perf_event *event = file->private_data;
3670        struct ring_buffer *rb;
3671        unsigned int events = POLLHUP;
3672
3673        poll_wait(file, &event->waitq, wait);
3674
3675        if (is_event_hup(event))
3676                return events;
3677
3678        /*
3679         * Pin the event->rb by taking event->mmap_mutex; otherwise
3680         * perf_event_set_output() can swizzle our rb and make us miss wakeups.
3681         */
3682        mutex_lock(&event->mmap_mutex);
3683        rb = event->rb;
3684        if (rb)
3685                events = atomic_xchg(&rb->poll, 0);
3686        mutex_unlock(&event->mmap_mutex);
3687        return events;
3688}
3689
3690static void perf_event_reset(struct perf_event *event)
3691{
3692        (void)perf_event_read(event);
3693        local64_set(&event->count, 0);
3694        perf_event_update_userpage(event);
3695}
3696
3697/*
3698 * Holding the top-level event's child_mutex means that any
3699 * descendant process that has inherited this event will block
3700 * in sync_child_event if it goes to exit, thus satisfying the
3701 * task existence requirements of perf_event_enable/disable.
3702 */
3703static void perf_event_for_each_child(struct perf_event *event,
3704                                        void (*func)(struct perf_event *))
3705{
3706        struct perf_event *child;
3707
3708        WARN_ON_ONCE(event->ctx->parent_ctx);
3709        mutex_lock(&event->child_mutex);
3710        func(event);
3711        list_for_each_entry(child, &event->child_list, child_list)
3712                func(child);
3713        mutex_unlock(&event->child_mutex);
3714}
3715
3716static void perf_event_for_each(struct perf_event *event,
3717                                  void (*func)(struct perf_event *))
3718{
3719        struct perf_event_context *ctx = event->ctx;
3720        struct perf_event *sibling;
3721
3722        WARN_ON_ONCE(ctx->parent_ctx);
3723        mutex_lock(&ctx->mutex);
3724        event = event->group_leader;
3725
3726        perf_event_for_each_child(event, func);
3727        list_for_each_entry(sibling, &event->sibling_list, group_entry)
3728                perf_event_for_each_child(sibling, func);
3729        mutex_unlock(&ctx->mutex);
3730}
3731
3732struct period_event {
3733        struct perf_event *event;
3734        u64 value;
3735};
3736
3737static int __perf_event_period(void *info)
3738{
3739        struct period_event *pe = info;
3740        struct perf_event *event = pe->event;
3741        struct perf_event_context *ctx = event->ctx;
3742        u64 value = pe->value;
3743        bool active;
3744
3745        raw_spin_lock(&ctx->lock);
3746        if (event->attr.freq) {
3747                event->attr.sample_freq = value;
3748        } else {
3749                event->attr.sample_period = value;
3750                event->hw.sample_period = value;
3751        }
3752
3753        active = (event->state == PERF_EVENT_STATE_ACTIVE);
3754        if (active) {
3755                perf_pmu_disable(ctx->pmu);
3756                event->pmu->stop(event, PERF_EF_UPDATE);
3757        }
3758
3759        local64_set(&event->hw.period_left, 0);
3760
3761        if (active) {
3762                event->pmu->start(event, PERF_EF_RELOAD);
3763                perf_pmu_enable(ctx->pmu);
3764        }
3765        raw_spin_unlock(&ctx->lock);
3766
3767        return 0;
3768}
3769
3770static int perf_event_period(struct perf_event *event, u64 __user *arg)
3771{
3772        struct period_event pe = { .event = event, };
3773        struct perf_event_context *ctx = event->ctx;
3774        struct task_struct *task;
3775        u64 value;
3776
3777        if (!is_sampling_event(event))
3778                return -EINVAL;
3779
3780        if (copy_from_user(&value, arg, sizeof(value)))
3781                return -EFAULT;
3782
3783        if (!value)
3784                return -EINVAL;
3785
3786        if (event->attr.freq && value > sysctl_perf_event_sample_rate)
3787                return -EINVAL;
3788
3789        task = ctx->task;
3790        pe.value = value;
3791
3792        if (!task) {
3793                cpu_function_call(event->cpu, __perf_event_period, &pe);
3794                return 0;
3795        }
3796
3797retry:
3798        if (!task_function_call(task, __perf_event_period, &pe))
3799                return 0;
3800
3801        raw_spin_lock_irq(&ctx->lock);
3802        if (ctx->is_active) {
3803                raw_spin_unlock_irq(&ctx->lock);
3804                task = ctx->task;
3805                goto retry;
3806        }
3807
3808        __perf_event_period(&pe);
3809        raw_spin_unlock_irq(&ctx->lock);
3810
3811        return 0;
3812}
3813
3814static const struct file_operations perf_fops;
3815
3816static inline int perf_fget_light(int fd, struct fd *p)
3817{
3818        struct fd f = fdget(fd);
3819        if (!f.file)
3820                return -EBADF;
3821
3822        if (f.file->f_op != &perf_fops) {
3823                fdput(f);
3824                return -EBADF;
3825        }
3826        *p = f;
3827        return 0;
3828}
3829
3830static int perf_event_set_output(struct perf_event *event,
3831                                 struct perf_event *output_event);
3832static int perf_event_set_filter(struct perf_event *event, void __user *arg);
3833
3834static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
3835{
3836        struct perf_event *event = file->private_data;
3837        void (*func)(struct perf_event *);
3838        u32 flags = arg;
3839
3840        switch (cmd) {
3841        case PERF_EVENT_IOC_ENABLE:
3842                func = perf_event_enable;
3843                break;
3844        case PERF_EVENT_IOC_DISABLE:
3845                func = perf_event_disable;
3846                break;
3847        case PERF_EVENT_IOC_RESET:
3848                func = perf_event_reset;
3849                break;
3850
3851        case PERF_EVENT_IOC_REFRESH:
3852                return perf_event_refresh(event, arg);
3853
3854        case PERF_EVENT_IOC_PERIOD:
3855                return perf_event_period(event, (u64 __user *)arg);
3856
3857        case PERF_EVENT_IOC_ID:
3858        {
3859                u64 id = primary_event_id(event);
3860
3861                if (copy_to_user((void __user *)arg, &id, sizeof(id)))
3862                        return -EFAULT;
3863                return 0;
3864        }
3865
3866        case PERF_EVENT_IOC_SET_OUTPUT:
3867        {
3868                int ret;
3869                if (arg != -1) {
3870                        struct perf_event *output_event;
3871                        struct fd output;
3872                        ret = perf_fget_light(arg, &output);
3873                        if (ret)
3874                                return ret;
3875                        output_event = output.file->private_data;
3876                        ret = perf_event_set_output(event, output_event);
3877                        fdput(output);
3878                } else {
3879                        ret = perf_event_set_output(event, NULL);
3880                }
3881                return ret;
3882        }
3883
3884        case PERF_EVENT_IOC_SET_FILTER:
3885                return perf_event_set_filter(event, (void __user *)arg);
3886
3887        default:
3888                return -ENOTTY;
3889        }
3890
3891        if (flags & PERF_IOC_FLAG_GROUP)
3892                perf_event_for_each(event, func);
3893        else
3894                perf_event_for_each_child(event, func);
3895
3896        return 0;
3897}
3898
3899#ifdef CONFIG_COMPAT
3900static long perf_compat_ioctl(struct file *file, unsigned int cmd,
3901                                unsigned long arg)
3902{
3903        switch (_IOC_NR(cmd)) {
3904        case _IOC_NR(PERF_EVENT_IOC_SET_FILTER):
3905        case _IOC_NR(PERF_EVENT_IOC_ID):
3906                /* Fix up pointer size (usually 4 -> 8 in 32-on-64-bit case */
3907                if (_IOC_SIZE(cmd) == sizeof(compat_uptr_t)) {
3908                        cmd &= ~IOCSIZE_MASK;
3909                        cmd |= sizeof(void *) << IOCSIZE_SHIFT;
3910                }
3911                break;
3912        }
3913        return perf_ioctl(file, cmd, arg);
3914}
3915#else
3916# define perf_compat_ioctl NULL
3917#endif
3918
3919int perf_event_task_enable(void)
3920{
3921        struct perf_event *event;
3922
3923        mutex_lock(&current->perf_event_mutex);
3924        list_for_each_entry(event, &current->perf_event_list, owner_entry)
3925                perf_event_for_each_child(event, perf_event_enable);
3926        mutex_unlock(&current->perf_event_mutex);
3927
3928        return 0;
3929}
3930
3931int perf_event_task_disable(void)
3932{
3933        struct perf_event *event;
3934
3935        mutex_lock(&current->perf_event_mutex);
3936        list_for_each_entry(event, &current->perf_event_list, owner_entry)
3937                perf_event_for_each_child(event, perf_event_disable);
3938        mutex_unlock(&current->perf_event_mutex);
3939
3940        return 0;
3941}
3942
3943static int perf_event_index(struct perf_event *event)
3944{
3945        if (event->hw.state & PERF_HES_STOPPED)
3946                return 0;
3947
3948        if (event->state != PERF_EVENT_STATE_ACTIVE)
3949                return 0;
3950
3951        return event->pmu->event_idx(event);
3952}
3953
3954static void calc_timer_values(struct perf_event *event,
3955                                u64 *now,
3956                                u64 *enabled,
3957                                u64 *running)
3958{
3959        u64 ctx_time;
3960
3961        *now = perf_clock();
3962        ctx_time = event->shadow_ctx_time + *now;
3963        *enabled = ctx_time - event->tstamp_enabled;
3964        *running = ctx_time - event->tstamp_running;
3965}
3966
3967static void perf_event_init_userpage(struct perf_event *event)
3968{
3969        struct perf_event_mmap_page *userpg;
3970        struct ring_buffer *rb;
3971
3972        rcu_read_lock();
3973        rb = rcu_dereference(event->rb);
3974        if (!rb)
3975                goto unlock;
3976
3977        userpg = rb->user_page;
3978
3979        /* Allow new userspace to detect that bit 0 is deprecated */
3980        userpg->cap_bit0_is_deprecated = 1;
3981        userpg->size = offsetof(struct perf_event_mmap_page, __reserved);
3982
3983unlock:
3984        rcu_read_unlock();
3985}
3986
3987void __weak arch_perf_update_userpage(struct perf_event_mmap_page *userpg, u64 now)
3988{
3989}
3990
3991/*
3992 * Callers need to ensure there can be no nesting of this function, otherwise
3993 * the seqlock logic goes bad. We can not serialize this because the arch
3994 * code calls this from NMI context.
3995 */
3996void perf_event_update_userpage(struct perf_event *event)
3997{
3998        struct perf_event_mmap_page *userpg;
3999        struct ring_buffer *rb;
4000        u64 enabled, running, now;
4001
4002        rcu_read_lock();
4003        rb = rcu_dereference(event->rb);
4004        if (!rb)
4005                goto unlock;
4006
4007        /*
4008         * compute total_time_enabled, total_time_running
4009         * based on snapshot values taken when the event
4010         * was last scheduled in.
4011         *
4012         * we cannot simply called update_context_time()
4013         * because of locking issue as we can be called in
4014         * NMI context
4015         */
4016        calc_timer_values(event, &now, &enabled, &running);
4017
4018        userpg = rb->user_page;
4019        /*
4020         * Disable preemption so as to not let the corresponding user-space
4021         * spin too long if we get preempted.
4022         */
4023        preempt_disable();
4024        ++userpg->lock;
4025        barrier();
4026        userpg->index = perf_event_index(event);
4027        userpg->offset = perf_event_count(event);
4028        if (userpg->index)
4029                userpg->offset -= local64_read(&event->hw.prev_count);
4030
4031        userpg->time_enabled = enabled +
4032                        atomic64_read(&event->child_total_time_enabled);
4033
4034        userpg->time_running = running +
4035                        atomic64_read(&event->child_total_time_running);
4036
4037        arch_perf_update_userpage(userpg, now);
4038
4039        barrier();
4040        ++userpg->lock;
4041        preempt_enable();
4042unlock:
4043        rcu_read_unlock();
4044}
4045
4046static int perf_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
4047{
4048        struct perf_event *event = vma->vm_file->private_data;
4049        struct ring_buffer *rb;
4050        int ret = VM_FAULT_SIGBUS;
4051
4052        if (vmf->flags & FAULT_FLAG_MKWRITE) {
4053                if (vmf->pgoff == 0)
4054                        ret = 0;
4055                return ret;
4056        }
4057
4058        rcu_read_lock();
4059        rb = rcu_dereference(event->rb);
4060        if (!rb)
4061                goto unlock;
4062
4063        if (vmf->pgoff && (vmf->flags & FAULT_FLAG_WRITE))
4064                goto unlock;
4065
4066        vmf->page = perf_mmap_to_page(rb, vmf->pgoff);
4067        if (!vmf->page)
4068                goto unlock;
4069
4070        get_page(vmf->page);
4071        vmf->page->mapping = vma->vm_file->f_mapping;
4072        vmf->page->index   = vmf->pgoff;
4073
4074        ret = 0;
4075unlock:
4076        rcu_read_unlock();
4077
4078        return ret;
4079}
4080
4081static void ring_buffer_attach(struct perf_event *event,
4082                               struct ring_buffer *rb)
4083{
4084        struct ring_buffer *old_rb = NULL;
4085        unsigned long flags;
4086
4087        if (event->rb) {
4088                /*
4089                 * Should be impossible, we set this when removing
4090                 * event->rb_entry and wait/clear when adding event->rb_entry.
4091                 */
4092                WARN_ON_ONCE(event->rcu_pending);
4093
4094                old_rb = event->rb;
4095                spin_lock_irqsave(&old_rb->event_lock, flags);
4096                list_del_rcu(&event->rb_entry);
4097                spin_unlock_irqrestore(&old_rb->event_lock, flags);
4098
4099                event->rcu_batches = get_state_synchronize_rcu();
4100                event->rcu_pending = 1;
4101        }
4102
4103        if (rb) {
4104                if (event->rcu_pending) {
4105                        cond_synchronize_rcu(event->rcu_batches);
4106                        event->rcu_pending = 0;
4107                }
4108
4109                spin_lock_irqsave(&rb->event_lock, flags);
4110                list_add_rcu(&event->rb_entry, &rb->event_list);
4111                spin_unlock_irqrestore(&rb->event_lock, flags);
4112        }
4113
4114        rcu_assign_pointer(event->rb, rb);
4115
4116        if (old_rb) {
4117                ring_buffer_put(old_rb);
4118                /*
4119                 * Since we detached before setting the new rb, so that we
4120                 * could attach the new rb, we could have missed a wakeup.
4121                 * Provide it now.
4122                 */
4123                wake_up_all(&event->waitq);
4124        }
4125}
4126
4127static void ring_buffer_wakeup(struct perf_event *event)
4128{
4129        struct ring_buffer *rb;
4130
4131        rcu_read_lock();
4132        rb = rcu_dereference(event->rb);
4133        if (rb) {
4134                list_for_each_entry_rcu(event, &rb->event_list, rb_entry)
4135                        wake_up_all(&event->waitq);
4136        }
4137        rcu_read_unlock();
4138}
4139
4140static void rb_free_rcu(struct rcu_head *rcu_head)
4141{
4142        struct ring_buffer *rb;
4143
4144        rb = container_of(rcu_head, struct ring_buffer, rcu_head);
4145        rb_free(rb);
4146}
4147
4148static struct ring_buffer *ring_buffer_get(struct perf_event *event)
4149{
4150        struct ring_buffer *rb;
4151
4152        rcu_read_lock();
4153        rb = rcu_dereference(event->rb);
4154        if (rb) {
4155                if (!atomic_inc_not_zero(&rb->refcount))
4156                        rb = NULL;
4157        }
4158        rcu_read_unlock();
4159
4160        return rb;
4161}
4162
4163static void ring_buffer_put(struct ring_buffer *rb)
4164{
4165        if (!atomic_dec_and_test(&rb->refcount))
4166                return;
4167
4168        WARN_ON_ONCE(!list_empty(&rb->event_list));
4169
4170        call_rcu(&rb->rcu_head, rb_free_rcu);
4171}
4172
4173static void perf_mmap_open(struct vm_area_struct *vma)
4174{
4175        struct perf_event *event = vma->vm_file->private_data;
4176
4177        atomic_inc(&event->mmap_count);
4178        atomic_inc(&event->rb->mmap_count);
4179}
4180
4181/*
4182 * A buffer can be mmap()ed multiple times; either directly through the same
4183 * event, or through other events by use of perf_event_set_output().
4184 *
4185 * In order to undo the VM accounting done by perf_mmap() we need to destroy
4186 * the buffer here, where we still have a VM context. This means we need
4187 * to detach all events redirecting to us.
4188 */
4189static void perf_mmap_close(struct vm_area_struct *vma)
4190{
4191        struct perf_event *event = vma->vm_file->private_data;
4192
4193        struct ring_buffer *rb = ring_buffer_get(event);
4194        struct user_struct *mmap_user = rb->mmap_user;
4195        int mmap_locked = rb->mmap_locked;
4196        unsigned long size = perf_data_size(rb);
4197
4198        atomic_dec(&rb->mmap_count);
4199
4200        if (!atomic_dec_and_mutex_lock(&event->mmap_count, &event->mmap_mutex))
4201                goto out_put;
4202
4203        ring_buffer_attach(event, NULL);
4204        mutex_unlock(&event->mmap_mutex);
4205
4206        /* If there's still other mmap()s of this buffer, we're done. */
4207        if (atomic_read(&rb->mmap_count))
4208                goto out_put;
4209
4210        /*
4211         * No other mmap()s, detach from all other events that might redirect
4212         * into the now unreachable buffer. Somewhat complicated by the
4213         * fact that rb::event_lock otherwise nests inside mmap_mutex.
4214         */
4215again:
4216        rcu_read_lock();
4217        list_for_each_entry_rcu(event, &rb->event_list, rb_entry) {
4218                if (!atomic_long_inc_not_zero(&event->refcount)) {
4219                        /*
4220                         * This event is en-route to free_event() which will
4221                         * detach it and remove it from the list.
4222                         */
4223                        continue;
4224                }
4225                rcu_read_unlock();
4226
4227                mutex_lock(&event->mmap_mutex);
4228                /*
4229                 * Check we didn't race with perf_event_set_output() which can
4230                 * swizzle the rb from under us while we were waiting to
4231                 * acquire mmap_mutex.
4232                 *
4233                 * If we find a different rb; ignore this event, a next
4234                 * iteration will no longer find it on the list. We have to
4235                 * still restart the iteration to make sure we're not now
4236                 * iterating the wrong list.
4237                 */
4238                if (event->rb == rb)
4239                        ring_buffer_attach(event, NULL);
4240
4241                mutex_unlock(&event->mmap_mutex);
4242                put_event(event);
4243
4244                /*
4245                 * Restart the iteration; either we're on the wrong list or
4246                 * destroyed its integrity by doing a deletion.
4247                 */
4248                goto again;
4249        }
4250        rcu_read_unlock();
4251
4252        /*
4253         * It could be there's still a few 0-ref events on the list; they'll
4254         * get cleaned up by free_event() -- they'll also still have their
4255         * ref on the rb and will free it whenever they are done with it.
4256         *
4257         * Aside from that, this buffer is 'fully' detached and unmapped,
4258         * undo the VM accounting.
4259         */
4260
4261        atomic_long_sub((size >> PAGE_SHIFT) + 1, &mmap_user->locked_vm);
4262        vma->vm_mm->pinned_vm -= mmap_locked;
4263        free_uid(mmap_user);
4264
4265out_put:
4266        ring_buffer_put(rb); /* could be last */
4267}
4268
4269static const struct vm_operations_struct perf_mmap_vmops = {
4270        .open           = perf_mmap_open,
4271        .close          = perf_mmap_close,
4272        .fault          = perf_mmap_fault,
4273        .page_mkwrite   = perf_mmap_fault,
4274};
4275
4276static int perf_mmap(struct file *file, struct vm_area_struct *vma)
4277{
4278        struct perf_event *event = file->private_data;
4279        unsigned long user_locked, user_lock_limit;
4280        struct user_struct *user = current_user();
4281        unsigned long locked, lock_limit;
4282        struct ring_buffer *rb;
4283        unsigned long vma_size;
4284        unsigned long nr_pages;
4285        long user_extra, extra;
4286        int ret = 0, flags = 0;
4287
4288        /*
4289         * Don't allow mmap() of inherited per-task counters. This would
4290         * create a performance issue due to all children writing to the
4291         * same rb.
4292         */
4293        if (event->cpu == -1 && event->attr.inherit)
4294                return -EINVAL;
4295
4296        if (!(vma->vm_flags & VM_SHARED))
4297                return -EINVAL;
4298
4299        vma_size = vma->vm_end - vma->vm_start;
4300        nr_pages = (vma_size / PAGE_SIZE) - 1;
4301
4302        /*
4303         * If we have rb pages ensure they're a power-of-two number, so we
4304         * can do bitmasks instead of modulo.
4305         */
4306        if (nr_pages != 0 && !is_power_of_2(nr_pages))
4307                return -EINVAL;
4308
4309        if (vma_size != PAGE_SIZE * (1 + nr_pages))
4310                return -EINVAL;
4311
4312        if (vma->vm_pgoff != 0)
4313                return -EINVAL;
4314
4315        WARN_ON_ONCE(event->ctx->parent_ctx);
4316again:
4317        mutex_lock(&event->mmap_mutex);
4318        if (event->rb) {
4319                if (event->rb->nr_pages != nr_pages) {
4320                        ret = -EINVAL;
4321                        goto unlock;
4322                }
4323
4324                if (!atomic_inc_not_zero(&event->rb->mmap_count)) {
4325                        /*
4326                         * Raced against perf_mmap_close() through
4327                         * perf_event_set_output(). Try again, hope for better
4328                         * luck.
4329                         */
4330                        mutex_unlock(&event->mmap_mutex);
4331                        goto again;
4332                }
4333
4334                goto unlock;
4335        }
4336
4337        user_extra = nr_pages + 1;
4338        user_lock_limit = sysctl_perf_event_mlock >> (PAGE_SHIFT - 10);
4339
4340        /*
4341         * Increase the limit linearly with more CPUs:
4342         */
4343        user_lock_limit *= num_online_cpus();
4344
4345        user_locked = atomic_long_read(&user->locked_vm) + user_extra;
4346
4347        extra = 0;
4348        if (user_locked > user_lock_limit)
4349                extra = user_locked - user_lock_limit;
4350
4351        lock_limit = rlimit(RLIMIT_MEMLOCK);
4352        lock_limit >>= PAGE_SHIFT;
4353        locked = vma->vm_mm->pinned_vm + extra;
4354
4355        if ((locked > lock_limit) && perf_paranoid_tracepoint_raw() &&
4356                !capable(CAP_IPC_LOCK)) {
4357                ret = -EPERM;
4358                goto unlock;
4359        }
4360
4361        WARN_ON(event->rb);
4362
4363        if (vma->vm_flags & VM_WRITE)
4364                flags |= RING_BUFFER_WRITABLE;
4365
4366        rb = rb_alloc(nr_pages,
4367                event->attr.watermark ? event->attr.wakeup_watermark : 0,
4368                event->cpu, flags);
4369
4370        if (!rb) {
4371                ret = -ENOMEM;
4372                goto unlock;
4373        }
4374
4375        atomic_set(&rb->mmap_count, 1);
4376        rb->mmap_locked = extra;
4377        rb->mmap_user = get_current_user();
4378
4379        atomic_long_add(user_extra, &user->locked_vm);
4380        vma->vm_mm->pinned_vm += extra;
4381
4382        ring_buffer_attach(event, rb);
4383
4384        perf_event_init_userpage(event);
4385        perf_event_update_userpage(event);
4386
4387unlock:
4388        if (!ret)
4389                atomic_inc(&event->mmap_count);
4390        mutex_unlock(&event->mmap_mutex);
4391
4392        /*
4393         * Since pinned accounting is per vm we cannot allow fork() to copy our
4394         * vma.
4395         */
4396        vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_DONTDUMP;
4397        vma->vm_ops = &perf_mmap_vmops;
4398
4399        return ret;
4400}
4401
4402static int perf_fasync(int fd, struct file *filp, int on)
4403{
4404        struct inode *inode = file_inode(filp);
4405        struct perf_event *event = filp->private_data;
4406        int retval;
4407
4408        mutex_lock(&inode->i_mutex);
4409        retval = fasync_helper(fd, filp, on, &event->fasync);
4410        mutex_unlock(&inode->i_mutex);
4411
4412        if (retval < 0)
4413                return retval;
4414
4415        return 0;
4416}
4417
4418static const struct file_operations perf_fops = {
4419        .llseek                 = no_llseek,
4420        .release                = perf_release,
4421        .read                   = perf_read,
4422        .poll                   = perf_poll,
4423        .unlocked_ioctl         = perf_ioctl,
4424        .compat_ioctl           = perf_compat_ioctl,
4425        .mmap                   = perf_mmap,
4426        .fasync                 = perf_fasync,
4427};
4428
4429/*
4430 * Perf event wakeup
4431 *
4432 * If there's data, ensure we set the poll() state and publish everything
4433 * to user-space before waking everybody up.
4434 */
4435
4436static inline struct fasync_struct **perf_event_fasync(struct perf_event *event)
4437{
4438        /* only the parent has fasync state */
4439        if (event->parent)
4440                event = event->parent;
4441        return &event->fasync;
4442}
4443
4444void perf_event_wakeup(struct perf_event *event)
4445{
4446        ring_buffer_wakeup(event);
4447
4448        if (event->pending_kill) {
4449                kill_fasync(perf_event_fasync(event), SIGIO, event->pending_kill);
4450                event->pending_kill = 0;
4451        }
4452}
4453
4454static void perf_pending_event(struct irq_work *entry)
4455{
4456        struct perf_event *event = container_of(entry,
4457                        struct perf_event, pending);
4458        int rctx;
4459
4460        rctx = perf_swevent_get_recursion_context();
4461        /*
4462         * If we 'fail' here, that's OK, it means recursion is already disabled
4463         * and we won't recurse 'further'.
4464         */
4465
4466        if (event->pending_disable) {
4467                event->pending_disable = 0;
4468                __perf_event_disable(event);
4469        }
4470
4471        if (event->pending_wakeup) {
4472                event->pending_wakeup = 0;
4473                perf_event_wakeup(event);
4474        }
4475
4476        if (rctx >= 0)
4477                perf_swevent_put_recursion_context(rctx);
4478}
4479
4480/*
4481 * We assume there is only KVM supporting the callbacks.
4482 * Later on, we might change it to a list if there is
4483 * another virtualization implementation supporting the callbacks.
4484 */
4485struct perf_guest_info_callbacks *perf_guest_cbs;
4486
4487int perf_register_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4488{
4489        perf_guest_cbs = cbs;
4490        return 0;
4491}
4492EXPORT_SYMBOL_GPL(perf_register_guest_info_callbacks);
4493
4494int perf_unregister_guest_info_callbacks(struct perf_guest_info_callbacks *cbs)
4495{
4496        perf_guest_cbs = NULL;
4497        return 0;
4498}
4499EXPORT_SYMBOL_GPL(perf_unregister_guest_info_callbacks);
4500
4501static void
4502perf_output_sample_regs(struct perf_output_handle *handle,
4503                        struct pt_regs *regs, u64 mask)
4504{
4505        int bit;
4506
4507        for_each_set_bit(bit, (const unsigned long *) &mask,
4508                         sizeof(mask) * BITS_PER_BYTE) {
4509                u64 val;
4510
4511                val = perf_reg_value(regs, bit);
4512                perf_output_put(handle, val);
4513        }
4514}
4515
4516static void perf_sample_regs_user(struct perf_regs_user *regs_user,
4517                                  struct pt_regs *regs)
4518{
4519        if (!user_mode(regs)) {
4520                if (current->mm)
4521                        regs = task_pt_regs(current);
4522                else
4523                        regs = NULL;
4524        }
4525
4526        if (regs) {
4527                regs_user->regs = regs;
4528                regs_user->abi  = perf_reg_abi(current);
4529        }
4530}
4531
4532/*
4533 * Get remaining task size from user stack pointer.
4534 *
4535 * It'd be better to take stack vma map and limit this more
4536 * precisly, but there's no way to get it safely under interrupt,
4537 * so using TASK_SIZE as limit.
4538 */
4539static u64 perf_ustack_task_size(struct pt_regs *regs)
4540{
4541        unsigned long addr = perf_user_stack_pointer(regs);
4542
4543        if (!addr || addr >= TASK_SIZE)
4544                return 0;
4545
4546        return TASK_SIZE - addr;
4547}
4548
4549static u16
4550perf_sample_ustack_size(u16 stack_size, u16 header_size,
4551                        struct pt_regs *regs)
4552{
4553        u64 task_size;
4554
4555        /* No regs, no stack pointer, no dump. */
4556        if (!regs)
4557                return 0;
4558
4559        /*
4560         * Check if we fit in with the requested stack size into the:
4561         * - TASK_SIZE
4562         *   If we don't, we limit the size to the TASK_SIZE.
4563         *
4564         * - remaining sample size
4565         *   If we don't, we customize the stack size to
4566         *   fit in to the remaining sample size.
4567         */
4568
4569        task_size  = min((u64) USHRT_MAX, perf_ustack_task_size(regs));
4570        stack_size = min(stack_size, (u16) task_size);
4571
4572        /* Current header size plus static size and dynamic size. */
4573        header_size += 2 * sizeof(u64);
4574
4575        /* Do we fit in with the current stack dump size? */
4576        if ((u16) (header_size + stack_size) < header_size) {
4577                /*
4578                 * If we overflow the maximum size for the sample,
4579                 * we customize the stack dump size to fit in.
4580                 */
4581                stack_size = USHRT_MAX - header_size - sizeof(u64);
4582                stack_size = round_up(stack_size, sizeof(u64));
4583        }
4584
4585        return stack_size;
4586}
4587
4588static void
4589perf_output_sample_ustack(struct perf_output_handle *handle, u64 dump_size,
4590                          struct pt_regs *regs)
4591{
4592        /* Case of a kernel thread, nothing to dump */
4593        if (!regs) {
4594                u64 size = 0;
4595                perf_output_put(handle, size);
4596        } else {
4597                unsigned long sp;
4598                unsigned int rem;
4599                u64 dyn_size;
4600
4601                /*
4602                 * We dump:
4603                 * static size
4604                 *   - the size requested by user or the best one we can fit
4605                 *     in to the sample max size
4606                 * data
4607                 *   - user stack dump data
4608                 * dynamic size
4609                 *   - the actual dumped size
4610                 */
4611
4612                /* Static size. */
4613                perf_output_put(handle, dump_size);
4614
4615                /* Data. */
4616                sp = perf_user_stack_pointer(regs);
4617                rem = __output_copy_user(handle, (void *) sp, dump_size);
4618                dyn_size = dump_size - rem;
4619
4620                perf_output_skip(handle, rem);
4621
4622                /* Dynamic size. */
4623                perf_output_put(handle, dyn_size);
4624        }
4625}
4626
4627static void __perf_event_header__init_id(struct perf_event_header *header,
4628                                         struct perf_sample_data *data,
4629                                         struct perf_event *event)
4630{
4631        u64 sample_type = event->attr.sample_type;
4632
4633        data->type = sample_type;
4634        header->size += event->id_header_size;
4635
4636        if (sample_type & PERF_SAMPLE_TID) {
4637                /* namespace issues */
4638                data->tid_entry.pid = perf_event_pid(event, current);
4639                data->tid_entry.tid = perf_event_tid(event, current);
4640        }
4641
4642        if (sample_type & PERF_SAMPLE_TIME)
4643                data->time = perf_clock();
4644
4645        if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
4646                data->id = primary_event_id(event);
4647
4648        if (sample_type & PERF_SAMPLE_STREAM_ID)
4649                data->stream_id = event->id;
4650
4651        if (sample_type & PERF_SAMPLE_CPU) {
4652                data->cpu_entry.cpu      = raw_smp_processor_id();
4653                data->cpu_entry.reserved = 0;
4654        }
4655}
4656
4657void perf_event_header__init_id(struct perf_event_header *header,
4658                                struct perf_sample_data *data,
4659                                struct perf_event *event)
4660{
4661        if (event->attr.sample_id_all)
4662                __perf_event_header__init_id(header, data, event);
4663}
4664
4665static void __perf_event__output_id_sample(struct perf_output_handle *handle,
4666                                           struct perf_sample_data *data)
4667{
4668        u64 sample_type = data->type;
4669
4670        if (sample_type & PERF_SAMPLE_TID)
4671                perf_output_put(handle, data->tid_entry);
4672
4673        if (sample_type & PERF_SAMPLE_TIME)
4674                perf_output_put(handle, data->time);
4675
4676        if (sample_type & PERF_SAMPLE_ID)
4677                perf_output_put(handle, data->id);
4678
4679        if (sample_type & PERF_SAMPLE_STREAM_ID)
4680                perf_output_put(handle, data->stream_id);
4681
4682        if (sample_type & PERF_SAMPLE_CPU)
4683                perf_output_put(handle, data->cpu_entry);
4684
4685        if (sample_type & PERF_SAMPLE_IDENTIFIER)
4686                perf_output_put(handle, data->id);
4687}
4688
4689void perf_event__output_id_sample(struct perf_event *event,
4690                                  struct perf_output_handle *handle,
4691                                  struct perf_sample_data *sample)
4692{
4693        if (event->attr.sample_id_all)
4694                __perf_event__output_id_sample(handle, sample);
4695}
4696
4697static void perf_output_read_one(struct perf_output_handle *handle,
4698                                 struct perf_event *event,
4699                                 u64 enabled, u64 running)
4700{
4701        u64 read_format = event->attr.read_format;
4702        u64 values[4];
4703        int n = 0;
4704
4705        values[n++] = perf_event_count(event);
4706        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) {
4707                values[n++] = enabled +
4708                        atomic64_read(&event->child_total_time_enabled);
4709        }
4710        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) {
4711                values[n++] = running +
4712                        atomic64_read(&event->child_total_time_running);
4713        }
4714        if (read_format & PERF_FORMAT_ID)
4715                values[n++] = primary_event_id(event);
4716
4717        __output_copy(handle, values, n * sizeof(u64));
4718}
4719
4720/*
4721 * XXX PERF_FORMAT_GROUP vs inherited events seems difficult.
4722 */
4723static void perf_output_read_group(struct perf_output_handle *handle,
4724                            struct perf_event *event,
4725                            u64 enabled, u64 running)
4726{
4727        struct perf_event *leader = event->group_leader, *sub;
4728        u64 read_format = event->attr.read_format;
4729        u64 values[5];
4730        int n = 0;
4731
4732        values[n++] = 1 + leader->nr_siblings;
4733
4734        if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
4735                values[n++] = enabled;
4736
4737        if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
4738                values[n++] = running;
4739
4740        if (leader != event)
4741                leader->pmu->read(leader);
4742
4743        values[n++] = perf_event_count(leader);
4744        if (read_format & PERF_FORMAT_ID)
4745                values[n++] = primary_event_id(leader);
4746
4747        __output_copy(handle, values, n * sizeof(u64));
4748
4749        list_for_each_entry(sub, &leader->sibling_list, group_entry) {
4750                n = 0;
4751
4752                if ((sub != event) &&
4753                    (sub->state == PERF_EVENT_STATE_ACTIVE))
4754                        sub->pmu->read(sub);
4755
4756                values[n++] = perf_event_count(sub);
4757                if (read_format & PERF_FORMAT_ID)
4758                        values[n++] = primary_event_id(sub);
4759
4760                __output_copy(handle, values, n * sizeof(u64));
4761        }
4762}
4763
4764#define PERF_FORMAT_TOTAL_TIMES (PERF_FORMAT_TOTAL_TIME_ENABLED|\
4765                                 PERF_FORMAT_TOTAL_TIME_RUNNING)
4766
4767static void perf_output_read(struct perf_output_handle *handle,
4768                             struct perf_event *event)
4769{
4770        u64 enabled = 0, running = 0, now;
4771        u64 read_format = event->attr.read_format;
4772
4773        /*
4774         * compute total_time_enabled, total_time_running
4775         * based on snapshot values taken when the event
4776         * was last scheduled in.
4777         *
4778         * we cannot simply called update_context_time()
4779         * because of locking issue as we are called in
4780         * NMI context
4781         */
4782        if (read_format & PERF_FORMAT_TOTAL_TIMES)
4783                calc_timer_values(event, &now, &enabled, &running);
4784
4785        if (event->attr.read_format & PERF_FORMAT_GROUP)
4786                perf_output_read_group(handle, event, enabled, running);
4787        else
4788                perf_output_read_one(handle, event, enabled, running);
4789}
4790
4791void perf_output_sample(struct perf_output_handle *handle,
4792                        struct perf_event_header *header,
4793                        struct perf_sample_data *data,
4794                        struct perf_event *event)
4795{
4796        u64 sample_type = data->type;
4797
4798        perf_output_put(handle, *header);
4799
4800        if (sample_type & PERF_SAMPLE_IDENTIFIER)
4801                perf_output_put(handle, data->id);
4802
4803        if (sample_type & PERF_SAMPLE_IP)
4804                perf_output_put(handle, data->ip);
4805
4806        if (sample_type & PERF_SAMPLE_TID)
4807                perf_output_put(handle, data->tid_entry);
4808
4809        if (sample_type & PERF_SAMPLE_TIME)
4810                perf_output_put(handle, data->time);
4811
4812        if (sample_type & PERF_SAMPLE_ADDR)
4813                perf_output_put(handle, data->addr);
4814
4815        if (sample_type & PERF_SAMPLE_ID)
4816                perf_output_put(handle, data->id);
4817
4818        if (sample_type & PERF_SAMPLE_STREAM_ID)
4819                perf_output_put(handle, data->stream_id);
4820
4821        if (sample_type & PERF_SAMPLE_CPU)
4822                perf_output_put(handle, data->cpu_entry);
4823
4824        if (sample_type & PERF_SAMPLE_PERIOD)
4825                perf_output_put(handle, data->period);
4826
4827        if (sample_type & PERF_SAMPLE_READ)
4828                perf_output_read(handle, event);
4829
4830        if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4831                if (data->callchain) {
4832                        int size = 1;
4833
4834                        if (data->callchain)
4835                                size += data->callchain->nr;
4836
4837                        size *= sizeof(u64);
4838
4839                        __output_copy(handle, data->callchain, size);
4840                } else {
4841                        u64 nr = 0;
4842                        perf_output_put(handle, nr);
4843                }
4844        }
4845
4846        if (sample_type & PERF_SAMPLE_RAW) {
4847                if (data->raw) {
4848                        perf_output_put(handle, data->raw->size);
4849                        __output_copy(handle, data->raw->data,
4850                                           data->raw->size);
4851                } else {
4852                        struct {
4853                                u32     size;
4854                                u32     data;
4855                        } raw = {
4856                                .size = sizeof(u32),
4857                                .data = 0,
4858                        };
4859                        perf_output_put(handle, raw);
4860                }
4861        }
4862
4863        if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4864                if (data->br_stack) {
4865                        size_t size;
4866
4867                        size = data->br_stack->nr
4868                             * sizeof(struct perf_branch_entry);
4869
4870                        perf_output_put(handle, data->br_stack->nr);
4871                        perf_output_copy(handle, data->br_stack->entries, size);
4872                } else {
4873                        /*
4874                         * we always store at least the value of nr
4875                         */
4876                        u64 nr = 0;
4877                        perf_output_put(handle, nr);
4878                }
4879        }
4880
4881        if (sample_type & PERF_SAMPLE_REGS_USER) {
4882                u64 abi = data->regs_user.abi;
4883
4884                /*
4885                 * If there are no regs to dump, notice it through
4886                 * first u64 being zero (PERF_SAMPLE_REGS_ABI_NONE).
4887                 */
4888                perf_output_put(handle, abi);
4889
4890                if (abi) {
4891                        u64 mask = event->attr.sample_regs_user;
4892                        perf_output_sample_regs(handle,
4893                                                data->regs_user.regs,
4894                                                mask);
4895                }
4896        }
4897
4898        if (sample_type & PERF_SAMPLE_STACK_USER) {
4899                perf_output_sample_ustack(handle,
4900                                          data->stack_user_size,
4901                                          data->regs_user.regs);
4902        }
4903
4904        if (sample_type & PERF_SAMPLE_WEIGHT)
4905                perf_output_put(handle, data->weight);
4906
4907        if (sample_type & PERF_SAMPLE_DATA_SRC)
4908                perf_output_put(handle, data->data_src.val);
4909
4910        if (sample_type & PERF_SAMPLE_TRANSACTION)
4911                perf_output_put(handle, data->txn);
4912
4913        if (!event->attr.watermark) {
4914                int wakeup_events = event->attr.wakeup_events;
4915
4916                if (wakeup_events) {
4917                        struct ring_buffer *rb = handle->rb;
4918                        int events = local_inc_return(&rb->events);
4919
4920                        if (events >= wakeup_events) {
4921                                local_sub(wakeup_events, &rb->events);
4922                                local_inc(&rb->wakeup);
4923                        }
4924                }
4925        }
4926}
4927
4928void perf_prepare_sample(struct perf_event_header *header,
4929                         struct perf_sample_data *data,
4930                         struct perf_event *event,
4931                         struct pt_regs *regs)
4932{
4933        u64 sample_type = event->attr.sample_type;
4934
4935        header->type = PERF_RECORD_SAMPLE;
4936        header->size = sizeof(*header) + event->header_size;
4937
4938        header->misc = 0;
4939        header->misc |= perf_misc_flags(regs);
4940
4941        __perf_event_header__init_id(header, data, event);
4942
4943        if (sample_type & PERF_SAMPLE_IP)
4944                data->ip = perf_instruction_pointer(regs);
4945
4946        if (sample_type & PERF_SAMPLE_CALLCHAIN) {
4947                int size = 1;
4948
4949                data->callchain = perf_callchain(event, regs);
4950
4951                if (data->callchain)
4952                        size += data->callchain->nr;
4953
4954                header->size += size * sizeof(u64);
4955        }
4956
4957        if (sample_type & PERF_SAMPLE_RAW) {
4958                int size = sizeof(u32);
4959
4960                if (data->raw)
4961                        size += data->raw->size;
4962                else
4963                        size += sizeof(u32);
4964
4965                WARN_ON_ONCE(size & (sizeof(u64)-1));
4966                header->size += size;
4967        }
4968
4969        if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
4970                int size = sizeof(u64); /* nr */
4971                if (data->br_stack) {
4972                        size += data->br_stack->nr
4973                              * sizeof(struct perf_branch_entry);
4974                }
4975                header->size += size;
4976        }
4977
4978        if (sample_type & PERF_SAMPLE_REGS_USER) {
4979                /* regs dump ABI info */
4980                int size = sizeof(u64);
4981
4982                perf_sample_regs_user(&data->regs_user, regs);
4983
4984                if (data->regs_user.regs) {
4985                        u64 mask = event->attr.sample_regs_user;
4986                        size += hweight64(mask) * sizeof(u64);
4987                }
4988
4989                header->size += size;
4990        }
4991
4992        if (sample_type & PERF_SAMPLE_STACK_USER) {
4993                /*
4994                 * Either we need PERF_SAMPLE_STACK_USER bit to be allways
4995                 * processed as the last one or have additional check added
4996                 * in case new sample type is added, because we could eat
4997                 * up the rest of the sample size.
4998                 */
4999                struct perf_regs_user *uregs = &data->regs_user;
5000                u16 stack_size = event->attr.sample_stack_user;
5001                u16 size = sizeof(u64);
5002
5003                if (!uregs->abi)
5004                        perf_sample_regs_user(uregs, regs);
5005
5006                stack_size = perf_sample_ustack_size(stack_size, header->size,
5007                                                     uregs->regs);
5008
5009                /*
5010                 * If there is something to dump, add space for the dump
5011                 * itself and for the field that tells the dynamic size,
5012                 * which is how many have been actually dumped.
5013                 */
5014                if (stack_size)
5015                        size += sizeof(u64) + stack_size;
5016
5017                data->stack_user_size = stack_size;
5018                header->size += size;
5019        }
5020}
5021
5022static void perf_event_output(struct perf_event *event,
5023                                struct perf_sample_data *data,
5024                                struct pt_regs *regs)
5025{
5026        struct perf_output_handle handle;
5027        struct perf_event_header header;
5028
5029        /* protect the callchain buffers */
5030        rcu_read_lock();
5031
5032        perf_prepare_sample(&header, data, event, regs);
5033
5034        if (perf_output_begin(&handle, event, header.size))
5035                goto exit;
5036
5037        perf_output_sample(&handle, &header, data, event);
5038
5039        perf_output_end(&handle);
5040
5041exit:
5042        rcu_read_unlock();
5043}
5044
5045/*
5046 * read event_id
5047 */
5048
5049struct perf_read_event {
5050        struct perf_event_header        header;
5051
5052        u32                             pid;
5053        u32                             tid;
5054};
5055
5056static void
5057perf_event_read_event(struct perf_event *event,
5058                        struct task_struct *task)
5059{
5060        struct perf_output_handle handle;
5061        struct perf_sample_data sample;
5062        struct perf_read_event read_event = {
5063                .header = {
5064                        .type = PERF_RECORD_READ,
5065                        .misc = 0,
5066                        .size = sizeof(read_event) + event->read_size,
5067                },
5068                .pid = perf_event_pid(event, task),
5069                .tid = perf_event_tid(event, task),
5070        };
5071        int ret;
5072
5073        perf_event_header__init_id(&read_event.header, &sample, event);
5074        ret = perf_output_begin(&handle, event, read_event.header.size);
5075        if (ret)
5076                return;
5077
5078        perf_output_put(&handle, read_event);
5079        perf_output_read(&handle, event);
5080        perf_event__output_id_sample(event, &handle, &sample);
5081
5082        perf_output_end(&handle);
5083}
5084
5085typedef void (perf_event_aux_output_cb)(struct perf_event *event, void *data);
5086
5087static void
5088perf_event_aux_ctx(struct perf_event_context *ctx,
5089                   perf_event_aux_output_cb output,
5090                   void *data)
5091{
5092        struct perf_event *event;
5093
5094        list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
5095                if (event->state < PERF_EVENT_STATE_INACTIVE)
5096                        continue;
5097                if (!event_filter_match(event))
5098                        continue;
5099                output(event, data);
5100        }
5101}
5102
5103static void
5104perf_event_aux(perf_event_aux_output_cb output, void *data,
5105               struct perf_event_context *task_ctx)
5106{
5107        struct perf_cpu_context *cpuctx;
5108        struct perf_event_context *ctx;
5109        struct pmu *pmu;
5110        int ctxn;
5111
5112        rcu_read_lock();
5113        list_for_each_entry_rcu(pmu, &pmus, entry) {
5114                cpuctx = get_cpu_ptr(pmu->pmu_cpu_context);
5115                if (cpuctx->unique_pmu != pmu)
5116                        goto next;
5117                perf_event_aux_ctx(&cpuctx->ctx, output, data);
5118                if (task_ctx)
5119                        goto next;
5120                ctxn = pmu->task_ctx_nr;
5121                if (ctxn < 0)
5122                        goto next;
5123                ctx = rcu_dereference(current->perf_event_ctxp[ctxn]);
5124                if (ctx)
5125                        perf_event_aux_ctx(ctx, output, data);
5126next:
5127                put_cpu_ptr(pmu->pmu_cpu_context);
5128        }
5129
5130        if (task_ctx) {
5131                preempt_disable();
5132                perf_event_aux_ctx(task_ctx, output, data);
5133                preempt_enable();
5134        }
5135        rcu_read_unlock();
5136}
5137
5138/*
5139 * task tracking -- fork/exit
5140 *
5141 * enabled by: attr.comm | attr.mmap | attr.mmap2 | attr.mmap_data | attr.task
5142 */
5143
5144struct perf_task_event {
5145        struct task_struct              *task;
5146        struct perf_event_context       *task_ctx;
5147
5148        struct {
5149                struct perf_event_header        header;
5150
5151                u32                             pid;
5152                u32                             ppid;
5153                u32                             tid;
5154                u32                             ptid;
5155                u64                             time;
5156        } event_id;
5157};
5158
5159static int perf_event_task_match(struct perf_event *event)
5160{
5161        return event->attr.comm  || event->attr.mmap ||
5162               event->attr.mmap2 || event->attr.mmap_data ||
5163               event->attr.task;
5164}
5165
5166static void perf_event_task_output(struct perf_event *event,
5167                                   void *data)
5168{
5169        struct perf_task_event *task_event = data;
5170        struct perf_output_handle handle;
5171        struct perf_sample_data sample;
5172        struct task_struct *task = task_event->task;
5173        int ret, size = task_event->event_id.header.size;
5174
5175        if (!perf_event_task_match(event))
5176                return;
5177
5178        perf_event_header__init_id(&task_event->event_id.header, &sample, event);
5179
5180        ret = perf_output_begin(&handle, event,
5181                                task_event->event_id.header.size);
5182        if (ret)
5183                goto out;
5184
5185        task_event->event_id.pid = perf_event_pid(event, task);
5186        task_event->event_id.ppid = perf_event_pid(event, current);
5187
5188        task_event->event_id.tid = perf_event_tid(event, task);
5189        task_event->event_id.ptid = perf_event_tid(event, current);
5190
5191        perf_output_put(&handle, task_event->event_id);
5192
5193        perf_event__output_id_sample(event, &handle, &sample);
5194
5195        perf_output_end(&handle);
5196out:
5197        task_event->event_id.header.size = size;
5198}
5199
5200static void perf_event_task(struct task_struct *task,
5201                              struct perf_event_context *task_ctx,
5202                              int new)
5203{
5204        struct perf_task_event task_event;
5205
5206        if (!atomic_read(&nr_comm_events) &&
5207            !atomic_read(&nr_mmap_events) &&
5208            !atomic_read(&nr_task_events))
5209                return;
5210
5211        task_event = (struct perf_task_event){
5212                .task     = task,
5213                .task_ctx = task_ctx,
5214                .event_id    = {
5215                        .header = {
5216                                .type = new ? PERF_RECORD_FORK : PERF_RECORD_EXIT,
5217                                .misc = 0,
5218                                .size = sizeof(task_event.event_id),
5219                        },
5220                        /* .pid  */
5221                        /* .ppid */
5222                        /* .tid  */
5223                        /* .ptid */
5224                        .time = perf_clock(),
5225                },
5226        };
5227
5228        perf_event_aux(perf_event_task_output,
5229                       &task_event,
5230                       task_ctx);
5231}
5232
5233void perf_event_fork(struct task_struct *task)
5234{
5235        perf_event_task(task, NULL, 1);
5236}
5237
5238/*
5239 * comm tracking
5240 */
5241
5242struct perf_comm_event {
5243        struct task_struct      *task;
5244        char                    *comm;
5245        int                     comm_size;
5246
5247        struct {
5248                struct perf_event_header        header;
5249
5250                u32                             pid;
5251                u32                             tid;
5252        } event_id;
5253};
5254
5255static int perf_event_comm_match(struct perf_event *event)
5256{
5257        return event->attr.comm;
5258}
5259
5260static void perf_event_comm_output(struct perf_event *event,
5261                                   void *data)
5262{
5263        struct perf_comm_event *comm_event = data;
5264        struct perf_output_handle handle;
5265        struct perf_sample_data sample;
5266        int size = comm_event->event_id.header.size;
5267        int ret;
5268
5269        if (!perf_event_comm_match(event))
5270                return;
5271
5272        perf_event_header__init_id(&comm_event->event_id.header, &sample, event);
5273        ret = perf_output_begin(&handle, event,
5274                                comm_event->event_id.header.size);
5275
5276        if (ret)
5277                goto out;
5278
5279        comm_event->event_id.pid = perf_event_pid(event, comm_event->task);
5280        comm_event->event_id.tid = perf_event_tid(event, comm_event->task);
5281
5282        perf_output_put(&handle, comm_event->event_id);
5283        __output_copy(&handle, comm_event->comm,
5284                                   comm_event->comm_size);
5285
5286        perf_event__output_id_sample(event, &handle, &sample);
5287
5288        perf_output_end(&handle);
5289out:
5290        comm_event->event_id.header.size = size;
5291}
5292
5293static void perf_event_comm_event(struct perf_comm_event *comm_event)
5294{
5295        char comm[TASK_COMM_LEN];
5296        unsigned int size;
5297
5298        memset(comm, 0, sizeof(comm));
5299        strlcpy(comm, comm_event->task->comm, sizeof(comm));
5300        size = ALIGN(strlen(comm)+1, sizeof(u64));
5301
5302        comm_event->comm = comm;
5303        comm_event->comm_size = size;
5304
5305        comm_event->event_id.header.size = sizeof(comm_event->event_id) + size;
5306
5307        perf_event_aux(perf_event_comm_output,
5308                       comm_event,
5309                       NULL);
5310}
5311
5312void perf_event_comm(struct task_struct *task, bool exec)
5313{
5314        struct perf_comm_event comm_event;
5315
5316        if (!atomic_read(&nr_comm_events))
5317                return;
5318
5319        comm_event = (struct perf_comm_event){
5320                .task   = task,
5321                /* .comm      */
5322                /* .comm_size */
5323                .event_id  = {
5324                        .header = {
5325                                .type = PERF_RECORD_COMM,
5326                                .misc = exec ? PERF_RECORD_MISC_COMM_EXEC : 0,
5327                                /* .size */
5328                        },
5329                        /* .pid */
5330                        /* .tid */
5331                },
5332        };
5333
5334        perf_event_comm_event(&comm_event);
5335}
5336
5337/*
5338 * mmap tracking
5339 */
5340
5341struct perf_mmap_event {
5342        struct vm_area_struct   *vma;
5343
5344        const char              *file_name;
5345        int                     file_size;
5346        int                     maj, min;
5347        u64                     ino;
5348        u64                     ino_generation;
5349        u32                     prot, flags;
5350
5351        struct {
5352                struct perf_event_header        header;
5353
5354                u32                             pid;
5355                u32                             tid;
5356                u64                             start;
5357                u64                             len;
5358                u64                             pgoff;
5359        } event_id;
5360};
5361
5362static int perf_event_mmap_match(struct perf_event *event,
5363                                 void *data)
5364{
5365        struct perf_mmap_event *mmap_event = data;
5366        struct vm_area_struct *vma = mmap_event->vma;
5367        int executable = vma->vm_flags & VM_EXEC;
5368
5369        return (!executable && event->attr.mmap_data) ||
5370               (executable && (event->attr.mmap || event->attr.mmap2));
5371}
5372
5373static void perf_event_mmap_output(struct perf_event *event,
5374                                   void *data)
5375{
5376        struct perf_mmap_event *mmap_event = data;
5377        struct perf_output_handle handle;
5378        struct perf_sample_data sample;
5379        int size = mmap_event->event_id.header.size;
5380        int ret;
5381
5382        if (!perf_event_mmap_match(event, data))
5383                return;
5384
5385        if (event->attr.mmap2) {
5386                mmap_event->event_id.header.type = PERF_RECORD_MMAP2;
5387                mmap_event->event_id.header.size += sizeof(mmap_event->maj);
5388                mmap_event->event_id.header.size += sizeof(mmap_event->min);
5389                mmap_event->event_id.header.size += sizeof(mmap_event->ino);
5390                mmap_event->event_id.header.size += sizeof(mmap_event->ino_generation);
5391                mmap_event->event_id.header.size += sizeof(mmap_event->prot);
5392                mmap_event->event_id.header.size += sizeof(mmap_event->flags);
5393        }
5394
5395        perf_event_header__init_id(&mmap_event->event_id.header, &sample, event);
5396        ret = perf_output_begin(&handle, event,
5397                                mmap_event->event_id.header.size);
5398        if (ret)
5399                goto out;
5400
5401        mmap_event->event_id.pid = perf_event_pid(event, current);
5402        mmap_event->event_id.tid = perf_event_tid(event, current);
5403
5404        perf_output_put(&handle, mmap_event->event_id);
5405
5406        if (event->attr.mmap2) {
5407                perf_output_put(&handle, mmap_event->maj);
5408                perf_output_put(&handle, mmap_event->min);
5409                perf_output_put(&handle, mmap_event->ino);
5410                perf_output_put(&handle, mmap_event->ino_generation);
5411                perf_output_put(&handle, mmap_event->prot);
5412                perf_output_put(&handle, mmap_event->flags);
5413        }
5414
5415        __output_copy(&handle, mmap_event->file_name,
5416                                   mmap_event->file_size);
5417
5418        perf_event__output_id_sample(event, &handle, &sample);
5419
5420        perf_output_end(&handle);
5421out:
5422        mmap_event->event_id.header.size = size;
5423}
5424
5425static void perf_event_mmap_event(struct perf_mmap_event *mmap_event)
5426{
5427        struct vm_area_struct *vma = mmap_event->vma;
5428        struct file *file = vma->vm_file;
5429        int maj = 0, min = 0;
5430        u64 ino = 0, gen = 0;
5431        u32 prot = 0, flags = 0;
5432        unsigned int size;
5433        char tmp[16];
5434        char *buf = NULL;
5435        char *name;
5436
5437        if (file) {
5438                struct inode *inode;
5439                dev_t dev;
5440
5441                buf = kmalloc(PATH_MAX, GFP_KERNEL);
5442                if (!buf) {
5443                        name = "//enomem";
5444                        goto cpy_name;
5445                }
5446                /*
5447                 * d_path() works from the end of the rb backwards, so we
5448                 * need to add enough zero bytes after the string to handle
5449                 * the 64bit alignment we do later.
5450                 */
5451                name = d_path(&file->f_path, buf, PATH_MAX - sizeof(u64));
5452                if (IS_ERR(name)) {
5453                        name = "//toolong";
5454                        goto cpy_name;
5455                }
5456                inode = file_inode(vma->vm_file);
5457                dev = inode->i_sb->s_dev;
5458                ino = inode->i_ino;
5459                gen = inode->i_generation;
5460                maj = MAJOR(dev);
5461                min = MINOR(dev);
5462
5463                if (vma->vm_flags & VM_READ)
5464                        prot |= PROT_READ;
5465                if (vma->vm_flags & VM_WRITE)
5466                        prot |= PROT_WRITE;
5467                if (vma->vm_flags & VM_EXEC)
5468                        prot |= PROT_EXEC;
5469
5470                if (vma->vm_flags & VM_MAYSHARE)
5471                        flags = MAP_SHARED;
5472                else
5473                        flags = MAP_PRIVATE;
5474
5475                if (vma->vm_flags & VM_DENYWRITE)
5476                        flags |= MAP_DENYWRITE;
5477                if (vma->vm_flags & VM_MAYEXEC)
5478                        flags |= MAP_EXECUTABLE;
5479                if (vma->vm_flags & VM_LOCKED)
5480                        flags |= MAP_LOCKED;
5481                if (vma->vm_flags & VM_HUGETLB)
5482                        flags |= MAP_HUGETLB;
5483
5484                goto got_name;
5485        } else {
5486                if (vma->vm_ops && vma->vm_ops->name) {
5487                        name = (char *) vma->vm_ops->name(vma);
5488                        if (name)
5489                                goto cpy_name;
5490                }
5491
5492                name = (char *)arch_vma_name(vma);
5493                if (name)
5494                        goto cpy_name;
5495
5496                if (vma->vm_start <= vma->vm_mm->start_brk &&
5497                                vma->vm_end >= vma->vm_mm->brk) {
5498                        name = "[heap]";
5499                        goto cpy_name;
5500                }
5501                if (vma->vm_start <= vma->vm_mm->start_stack &&
5502                                vma->vm_end >= vma->vm_mm->start_stack) {
5503                        name = "[stack]";
5504                        goto cpy_name;
5505                }
5506
5507                name = "//anon";
5508                goto cpy_name;
5509        }
5510
5511cpy_name:
5512        strlcpy(tmp, name, sizeof(tmp));
5513        name = tmp;
5514got_name:
5515        /*
5516         * Since our buffer works in 8 byte units we need to align our string
5517         * size to a multiple of 8. However, we must guarantee the tail end is
5518         * zero'd out to avoid leaking random bits to userspace.
5519         */
5520        size = strlen(name)+1;
5521        while (!IS_ALIGNED(size, sizeof(u64)))
5522                name[size++] = '\0';
5523
5524        mmap_event->file_name = name;
5525        mmap_event->file_size = size;
5526        mmap_event->maj = maj;
5527        mmap_event->min = min;
5528        mmap_event->ino = ino;
5529        mmap_event->ino_generation = gen;
5530        mmap_event->prot = prot;
5531        mmap_event->flags = flags;
5532
5533        if (!(vma->vm_flags & VM_EXEC))
5534                mmap_event->event_id.header.misc |= PERF_RECORD_MISC_MMAP_DATA;
5535
5536        mmap_event->event_id.header.size = sizeof(mmap_event->event_id) + size;
5537
5538        perf_event_aux(perf_event_mmap_output,
5539                       mmap_event,
5540                       NULL);
5541
5542        kfree(buf);
5543}
5544
5545void perf_event_mmap(struct vm_area_struct *vma)
5546{
5547        struct perf_mmap_event mmap_event;
5548
5549        if (!atomic_read(&nr_mmap_events))
5550                return;
5551
5552        mmap_event = (struct perf_mmap_event){
5553                .vma    = vma,
5554                /* .file_name */
5555                /* .file_size */
5556                .event_id  = {
5557                        .header = {
5558                                .type = PERF_RECORD_MMAP,
5559                                .misc = PERF_RECORD_MISC_USER,
5560                                /* .size */
5561                        },
5562                        /* .pid */
5563                        /* .tid */
5564                        .start  = vma->vm_start,
5565                        .len    = vma->vm_end - vma->vm_start,
5566                        .pgoff  = (u64)vma->vm_pgoff << PAGE_SHIFT,
5567                },
5568                /* .maj (attr_mmap2 only) */
5569                /* .min (attr_mmap2 only) */
5570                /* .ino (attr_mmap2 only) */
5571                /* .ino_generation (attr_mmap2 only) */
5572                /* .prot (attr_mmap2 only) */
5573                /* .flags (attr_mmap2 only) */
5574        };
5575
5576        perf_event_mmap_event(&mmap_event);
5577}
5578
5579/*
5580 * IRQ throttle logging
5581 */
5582
5583static void perf_log_throttle(struct perf_event *event, int enable)
5584{
5585        struct perf_output_handle handle;
5586        struct perf_sample_data sample;
5587        int ret;
5588
5589        struct {
5590                struct perf_event_header        header;
5591                u64                             time;
5592                u64                             id;
5593                u64                             stream_id;
5594        } throttle_event = {
5595                .header = {
5596                        .type = PERF_RECORD_THROTTLE,
5597                        .misc = 0,
5598                        .size = sizeof(throttle_event),
5599                },
5600                .time           = perf_clock(),
5601                .id             = primary_event_id(event),
5602                .stream_id      = event->id,
5603        };
5604
5605        if (enable)
5606                throttle_event.header.type = PERF_RECORD_UNTHROTTLE;
5607
5608        perf_event_header__init_id(&throttle_event.header, &sample, event);
5609
5610        ret = perf_output_begin(&handle, event,
5611                                throttle_event.header.size);
5612        if (ret)
5613                return;
5614
5615        perf_output_put(&handle, throttle_event);
5616        perf_event__output_id_sample(event, &handle, &sample);
5617        perf_output_end(&handle);
5618}
5619
5620/*
5621 * Generic event overflow handling, sampling.
5622 */
5623
5624static int __perf_event_overflow(struct perf_event *event,
5625                                   int throttle, struct perf_sample_data *data,
5626                                   struct pt_regs *regs)
5627{
5628        int events = atomic_read(&event->event_limit);
5629        struct hw_perf_event *hwc = &event->hw;
5630        u64 seq;
5631        int ret = 0;
5632
5633        /*
5634         * Non-sampling counters might still use the PMI to fold short
5635         * hardware counters, ignore those.
5636         */
5637        if (unlikely(!is_sampling_event(event)))
5638                return 0;
5639
5640        seq = __this_cpu_read(perf_throttled_seq);
5641        if (seq != hwc->interrupts_seq) {
5642                hwc->interrupts_seq = seq;
5643                hwc->interrupts = 1;
5644        } else {
5645                hwc->interrupts++;
5646                if (unlikely(throttle
5647                             && hwc->interrupts >= max_samples_per_tick)) {
5648                        __this_cpu_inc(perf_throttled_count);
5649                        hwc->interrupts = MAX_INTERRUPTS;
5650                        perf_log_throttle(event, 0);
5651                        tick_nohz_full_kick();
5652                        ret = 1;
5653                }
5654        }
5655
5656        if (event->attr.freq) {
5657                u64 now = perf_clock();
5658                s64 delta = now - hwc->freq_time_stamp;
5659
5660                hwc->freq_time_stamp = now;
5661
5662                if (delta > 0 && delta < 2*TICK_NSEC)
5663                        perf_adjust_period(event, delta, hwc->last_period, true);
5664        }
5665
5666        /*
5667         * XXX event_limit might not quite work as expected on inherited
5668         * events
5669         */
5670
5671        event->pending_kill = POLL_IN;
5672        if (events && atomic_dec_and_test(&event->event_limit)) {
5673                ret = 1;
5674                event->pending_kill = POLL_HUP;
5675                event->pending_disable = 1;
5676                irq_work_queue(&event->pending);
5677        }
5678
5679        if (event->overflow_handler)
5680                event->overflow_handler(event, data, regs);
5681        else
5682                perf_event_output(event, data, regs);
5683
5684        if (*perf_event_fasync(event) && event->pending_kill) {
5685                event->pending_wakeup = 1;
5686                irq_work_queue(&event->pending);
5687        }
5688
5689        return ret;
5690}
5691
5692int perf_event_overflow(struct perf_event *event,
5693                          struct perf_sample_data *data,
5694                          struct pt_regs *regs)
5695{
5696        return __perf_event_overflow(event, 1, data, regs);
5697}
5698
5699/*
5700 * Generic software event infrastructure
5701 */
5702
5703struct swevent_htable {
5704        struct swevent_hlist            *swevent_hlist;
5705        struct mutex                    hlist_mutex;
5706        int                             hlist_refcount;
5707
5708        /* Recursion avoidance in each contexts */
5709        int                             recursion[PERF_NR_CONTEXTS];
5710
5711        /* Keeps track of cpu being initialized/exited */
5712        bool                            online;
5713};
5714
5715static DEFINE_PER_CPU(struct swevent_htable, swevent_htable);
5716
5717/*
5718 * We directly increment event->count and keep a second value in
5719 * event->hw.period_left to count intervals. This period event
5720 * is kept in the range [-sample_period, 0] so that we can use the
5721 * sign as trigger.
5722 */
5723
5724u64 perf_swevent_set_period(struct perf_event *event)
5725{
5726        struct hw_perf_event *hwc = &event->hw;
5727        u64 period = hwc->last_period;
5728        u64 nr, offset;
5729        s64 old, val;
5730
5731        hwc->last_period = hwc->sample_period;
5732
5733again:
5734        old = val = local64_read(&hwc->period_left);
5735        if (val < 0)
5736                return 0;
5737
5738        nr = div64_u64(period + val, period);
5739        offset = nr * period;
5740        val -= offset;
5741        if (local64_cmpxchg(&hwc->period_left, old, val) != old)
5742                goto again;
5743
5744        return nr;
5745}
5746
5747static void perf_swevent_overflow(struct perf_event *event, u64 overflow,
5748                                    struct perf_sample_data *data,
5749                                    struct pt_regs *regs)
5750{
5751        struct hw_perf_event *hwc = &event->hw;
5752        int throttle = 0;
5753
5754        if (!overflow)
5755                overflow = perf_swevent_set_period(event);
5756
5757        if (hwc->interrupts == MAX_INTERRUPTS)
5758                return;
5759
5760        for (; overflow; overflow--) {
5761                if (__perf_event_overflow(event, throttle,
5762                                            data, regs)) {
5763                        /*
5764                         * We inhibit the overflow from happening when
5765                         * hwc->interrupts == MAX_INTERRUPTS.
5766                         */
5767                        break;
5768                }
5769                throttle = 1;
5770        }
5771}
5772
5773static void perf_swevent_event(struct perf_event *event, u64 nr,
5774                               struct perf_sample_data *data,
5775                               struct pt_regs *regs)
5776{
5777        struct hw_perf_event *hwc = &event->hw;
5778
5779        local64_add(nr, &event->count);
5780
5781        if (!regs)
5782                return;
5783
5784        if (!is_sampling_event(event))
5785                return;
5786
5787        if ((event->attr.sample_type & PERF_SAMPLE_PERIOD) && !event->attr.freq) {
5788                data->period = nr;
5789                return perf_swevent_overflow(event, 1, data, regs);
5790        } else
5791                data->period = event->hw.last_period;
5792
5793        if (nr == 1 && hwc->sample_period == 1 && !event->attr.freq)
5794                return perf_swevent_overflow(event, 1, data, regs);
5795
5796        if (local64_add_negative(nr, &hwc->period_left))
5797                return;
5798
5799        perf_swevent_overflow(event, 0, data, regs);
5800}
5801
5802static int perf_exclude_event(struct perf_event *event,
5803                              struct pt_regs *regs)
5804{
5805        if (event->hw.state & PERF_HES_STOPPED)
5806                return 1;
5807
5808        if (regs) {
5809                if (event->attr.exclude_user && user_mode(regs))
5810                        return 1;
5811
5812                if (event->attr.exclude_kernel && !user_mode(regs))
5813                        return 1;
5814        }
5815
5816        return 0;
5817}
5818
5819static int perf_swevent_match(struct perf_event *event,
5820                                enum perf_type_id type,
5821                                u32 event_id,
5822                                struct perf_sample_data *data,
5823                                struct pt_regs *regs)
5824{
5825        if (event->attr.type != type)
5826                return 0;
5827
5828        if (event->attr.config != event_id)
5829                return 0;
5830
5831        if (perf_exclude_event(event, regs))
5832                return 0;
5833
5834        return 1;
5835}
5836
5837static inline u64 swevent_hash(u64 type, u32 event_id)
5838{
5839        u64 val = event_id | (type << 32);
5840
5841        return hash_64(val, SWEVENT_HLIST_BITS);
5842}
5843
5844static inline struct hlist_head *
5845__find_swevent_head(struct swevent_hlist *hlist, u64 type, u32 event_id)
5846{
5847        u64 hash = swevent_hash(type, event_id);
5848
5849        return &hlist->heads[hash];
5850}
5851
5852/* For the read side: events when they trigger */
5853static inline struct hlist_head *
5854find_swevent_head_rcu(struct swevent_htable *swhash, u64 type, u32 event_id)
5855{
5856        struct swevent_hlist *hlist;
5857
5858        hlist = rcu_dereference(swhash->swevent_hlist);
5859        if (!hlist)
5860                return NULL;
5861
5862        return __find_swevent_head(hlist, type, event_id);
5863}
5864
5865/* For the event head insertion and removal in the hlist */
5866static inline struct hlist_head *
5867find_swevent_head(struct swevent_htable *swhash, struct perf_event *event)
5868{
5869        struct swevent_hlist *hlist;
5870        u32 event_id = event->attr.config;
5871        u64 type = event->attr.type;
5872
5873        /*
5874         * Event scheduling is always serialized against hlist allocation
5875         * and release. Which makes the protected version suitable here.
5876         * The context lock guarantees that.
5877         */
5878        hlist = rcu_dereference_protected(swhash->swevent_hlist,
5879                                          lockdep_is_held(&event->ctx->lock));
5880        if (!hlist)
5881                return NULL;
5882
5883        return __find_swevent_head(hlist, type, event_id);
5884}
5885
5886static void do_perf_sw_event(enum perf_type_id type, u32 event_id,
5887                                    u64 nr,
5888                                    struct perf_sample_data *data,
5889                                    struct pt_regs *regs)
5890{
5891        struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5892        struct perf_event *event;
5893        struct hlist_head *head;
5894
5895        rcu_read_lock();
5896        head = find_swevent_head_rcu(swhash, type, event_id);
5897        if (!head)
5898                goto end;
5899
5900        hlist_for_each_entry_rcu(event, head, hlist_entry) {
5901                if (perf_swevent_match(event, type, event_id, data, regs))
5902                        perf_swevent_event(event, nr, data, regs);
5903        }
5904end:
5905        rcu_read_unlock();
5906}
5907
5908int perf_swevent_get_recursion_context(void)
5909{
5910        struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5911
5912        return get_recursion_context(swhash->recursion);
5913}
5914EXPORT_SYMBOL_GPL(perf_swevent_get_recursion_context);
5915
5916inline void perf_swevent_put_recursion_context(int rctx)
5917{
5918        struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5919
5920        put_recursion_context(swhash->recursion, rctx);
5921}
5922
5923void __perf_sw_event(u32 event_id, u64 nr, struct pt_regs *regs, u64 addr)
5924{
5925        struct perf_sample_data data;
5926        int rctx;
5927
5928        preempt_disable_notrace();
5929        rctx = perf_swevent_get_recursion_context();
5930        if (rctx < 0)
5931                return;
5932
5933        perf_sample_data_init(&data, addr, 0);
5934
5935        do_perf_sw_event(PERF_TYPE_SOFTWARE, event_id, nr, &data, regs);
5936
5937        perf_swevent_put_recursion_context(rctx);
5938        preempt_enable_notrace();
5939}
5940
5941static void perf_swevent_read(struct perf_event *event)
5942{
5943}
5944
5945static int perf_swevent_add(struct perf_event *event, int flags)
5946{
5947        struct swevent_htable *swhash = this_cpu_ptr(&swevent_htable);
5948        struct hw_perf_event *hwc = &event->hw;
5949        struct hlist_head *head;
5950
5951        if (is_sampling_event(event)) {
5952                hwc->last_period = hwc->sample_period;
5953                perf_swevent_set_period(event);
5954        }
5955
5956        hwc->state = !(flags & PERF_EF_START);
5957
5958        head = find_swevent_head(swhash, event);
5959        if (!head) {
5960                /*
5961                 * We can race with cpu hotplug code. Do not
5962                 * WARN if the cpu just got unplugged.
5963                 */
5964                WARN_ON_ONCE(swhash->online);
5965                return -EINVAL;
5966        }
5967
5968        hlist_add_head_rcu(&event->hlist_entry, head);
5969
5970        return 0;
5971}
5972
5973static void perf_swevent_del(struct perf_event *event, int flags)
5974{
5975        hlist_del_rcu(&event->hlist_entry);
5976}
5977
5978static void perf_swevent_start(struct perf_event *event, int flags)
5979{
5980        event->hw.state = 0;
5981}
5982
5983static void perf_swevent_stop(struct perf_event *event, int flags)
5984{
5985        event->hw.state = PERF_HES_STOPPED;
5986}
5987
5988/* Deref the hlist from the update side */
5989static inline struct swevent_hlist *
5990swevent_hlist_deref(struct swevent_htable *swhash)
5991{
5992        return rcu_dereference_protected(swhash->swevent_hlist,
5993                                         lockdep_is_held(&swhash->hlist_mutex));
5994}
5995
5996static void swevent_hlist_release(struct swevent_htable *swhash)
5997{
5998        struct swevent_hlist *hlist = swevent_hlist_deref(swhash);
5999
6000        if (!hlist)
6001                return;
6002
6003        RCU_INIT_POINTER(swhash->swevent_hlist, NULL);
6004        kfree_rcu(hlist, rcu_head);
6005}
6006
6007static void swevent_hlist_put_cpu(struct perf_event *event, int cpu)
6008{
6009        struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6010
6011        mutex_lock(&swhash->hlist_mutex);
6012
6013        if (!--swhash->hlist_refcount)
6014                swevent_hlist_release(swhash);
6015
6016        mutex_unlock(&swhash->hlist_mutex);
6017}
6018
6019static void swevent_hlist_put(struct perf_event *event)
6020{
6021        int cpu;
6022
6023        for_each_possible_cpu(cpu)
6024                swevent_hlist_put_cpu(event, cpu);
6025}
6026
6027static int swevent_hlist_get_cpu(struct perf_event *event, int cpu)
6028{
6029        struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
6030        int err = 0;
6031
6032        mutex_lock(&swhash->hlist_mutex);
6033
6034        if (!swevent_hlist_deref(swhash) && cpu_online(cpu)) {
6035                struct swevent_hlist *hlist;
6036
6037                hlist = kzalloc(sizeof(*hlist), GFP_KERNEL);
6038                if (!hlist) {
6039                        err = -ENOMEM;
6040                        goto exit;
6041                }
6042                rcu_assign_pointer(swhash->swevent_hlist, hlist);
6043        }
6044        swhash->hlist_refcount++;
6045exit:
6046        mutex_unlock(&swhash->hlist_mutex);
6047
6048        return err;
6049}
6050
6051static int swevent_hlist_get(struct perf_event *event)
6052{
6053        int err;
6054        int cpu, failed_cpu;
6055
6056        get_online_cpus();
6057        for_each_possible_cpu(cpu) {
6058                err = swevent_hlist_get_cpu(event, cpu);
6059                if (err) {
6060                        failed_cpu = cpu;
6061                        goto fail;
6062                }
6063        }
6064        put_online_cpus();
6065
6066        return 0;
6067fail:
6068        for_each_possible_cpu(cpu) {
6069                if (cpu == failed_cpu)
6070                        break;
6071                swevent_hlist_put_cpu(event, cpu);
6072        }
6073
6074        put_online_cpus();
6075        return err;
6076}
6077
6078struct static_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
6079
6080static void sw_perf_event_destroy(struct perf_event *event)
6081{
6082        u64 event_id = event->attr.config;
6083
6084        WARN_ON(event->parent);
6085
6086        static_key_slow_dec(&perf_swevent_enabled[event_id]);
6087        swevent_hlist_put(event);
6088}
6089
6090static int perf_swevent_init(struct perf_event *event)
6091{
6092        u64 event_id = event->attr.config;
6093
6094        if (event->attr.type != PERF_TYPE_SOFTWARE)
6095                return -ENOENT;
6096
6097        /*
6098         * no branch sampling for software events
6099         */
6100        if (has_branch_stack(event))
6101                return -EOPNOTSUPP;
6102
6103        switch (event_id) {
6104        case PERF_COUNT_SW_CPU_CLOCK:
6105        case PERF_COUNT_SW_TASK_CLOCK:
6106                return -ENOENT;
6107
6108        default:
6109                break;
6110        }
6111
6112        if (event_id >= PERF_COUNT_SW_MAX)
6113                return -ENOENT;
6114
6115        if (!event->parent) {
6116                int err;
6117
6118                err = swevent_hlist_get(event);
6119                if (err)
6120                        return err;
6121
6122                static_key_slow_inc(&perf_swevent_enabled[event_id]);
6123                event->destroy = sw_perf_event_destroy;
6124        }
6125
6126        return 0;
6127}
6128
6129static struct pmu perf_swevent = {
6130        .task_ctx_nr    = perf_sw_context,
6131
6132        .event_init     = perf_swevent_init,
6133        .add            = perf_swevent_add,
6134        .del            = perf_swevent_del,
6135        .start          = perf_swevent_start,
6136        .stop           = perf_swevent_stop,
6137        .read           = perf_swevent_read,
6138};
6139
6140#ifdef CONFIG_EVENT_TRACING
6141
6142static int perf_tp_filter_match(struct perf_event *event,
6143                                struct perf_sample_data *data)
6144{
6145        void *record = data->raw->data;
6146
6147        if (likely(!event->filter) || filter_match_preds(event->filter, record))
6148                return 1;
6149        return 0;
6150}
6151
6152static int perf_tp_event_match(struct perf_event *event,
6153                                struct perf_sample_data *data,
6154                                struct pt_regs *regs)
6155{
6156        if (event->hw.state & PERF_HES_STOPPED)
6157                return 0;
6158        /*
6159         * All tracepoints are from kernel-space.
6160         */
6161        if (event->attr.exclude_kernel)
6162                return 0;
6163
6164        if (!perf_tp_filter_match(event, data))
6165                return 0;
6166
6167        return 1;
6168}
6169
6170void perf_tp_event(u64 addr, u64 count, void *record, int entry_size,
6171                   struct pt_regs *regs, struct hlist_head *head, int rctx,
6172                   struct task_struct *task)
6173{
6174        struct perf_sample_data data;
6175        struct perf_event *event;
6176
6177        struct perf_raw_record raw = {
6178                .size = entry_size,
6179                .data = record,
6180        };
6181
6182        perf_sample_data_init(&data, addr, 0);
6183        data.raw = &raw;
6184
6185        hlist_for_each_entry_rcu(event, head, hlist_entry) {
6186                if (perf_tp_event_match(event, &data, regs))
6187                        perf_swevent_event(event, count, &data, regs);
6188        }
6189
6190        /*
6191         * If we got specified a target task, also iterate its context and
6192         * deliver this event there too.
6193         */
6194        if (task && task != current) {
6195                struct perf_event_context *ctx;
6196                struct trace_entry *entry = record;
6197
6198                rcu_read_lock();
6199                ctx = rcu_dereference(task->perf_event_ctxp[perf_sw_context]);
6200                if (!ctx)
6201                        goto unlock;
6202
6203                list_for_each_entry_rcu(event, &ctx->event_list, event_entry) {
6204                        if (event->attr.type != PERF_TYPE_TRACEPOINT)
6205                                continue;
6206                        if (event->attr.config != entry->type)
6207                                continue;
6208                        if (perf_tp_event_match(event, &data, regs))
6209                                perf_swevent_event(event, count, &data, regs);
6210                }
6211unlock:
6212                rcu_read_unlock();
6213        }
6214
6215        perf_swevent_put_recursion_context(rctx);
6216}
6217EXPORT_SYMBOL_GPL(perf_tp_event);
6218
6219static void tp_perf_event_destroy(struct perf_event *event)
6220{
6221        perf_trace_destroy(event);
6222}
6223
6224static int perf_tp_event_init(struct perf_event *event)
6225{
6226        int err;
6227
6228        if (event->attr.type != PERF_TYPE_TRACEPOINT)
6229                return -ENOENT;
6230
6231        /*
6232         * no branch sampling for tracepoint events
6233         */
6234        if (has_branch_stack(event))
6235                return -EOPNOTSUPP;
6236
6237        err = perf_trace_init(event);
6238        if (err)
6239                return err;
6240
6241        event->destroy = tp_perf_event_destroy;
6242
6243        return 0;
6244}
6245
6246static struct pmu perf_tracepoint = {
6247        .task_ctx_nr    = perf_sw_context,
6248
6249        .event_init     = perf_tp_event_init,
6250        .add            = perf_trace_add,
6251        .del            = perf_trace_del,
6252        .start          = perf_swevent_start,
6253        .stop           = perf_swevent_stop,
6254        .read           = perf_swevent_read,
6255};
6256
6257static inline void perf_tp_register(void)
6258{
6259        perf_pmu_register(&perf_tracepoint, "tracepoint", PERF_TYPE_TRACEPOINT);
6260}
6261
6262static int perf_event_set_filter(struct perf_event *event, void __user *arg)
6263{
6264        char *filter_str;
6265        int ret;
6266
6267        if (event->attr.type != PERF_TYPE_TRACEPOINT)
6268                return -EINVAL;
6269
6270        filter_str = strndup_user(arg, PAGE_SIZE);
6271        if (IS_ERR(filter_str))
6272                return PTR_ERR(filter_str);
6273
6274        ret = ftrace_profile_set_filter(event, event->attr.config, filter_str);
6275
6276        kfree(filter_str);
6277        return ret;
6278}
6279
6280static void perf_event_free_filter(struct perf_event *event)
6281{
6282        ftrace_profile_free_filter(event);
6283}
6284
6285#else
6286
6287static inline void perf_tp_register(void)
6288{
6289}
6290
6291static int perf_event_set_filter(struct perf_event *event, void __user *arg)
6292{
6293        return -ENOENT;
6294}
6295
6296static void perf_event_free_filter(struct perf_event *event)
6297{
6298}
6299
6300#endif /* CONFIG_EVENT_TRACING */
6301
6302#ifdef CONFIG_HAVE_HW_BREAKPOINT
6303void perf_bp_event(struct perf_event *bp, void *data)
6304{
6305        struct perf_sample_data sample;
6306        struct pt_regs *regs = data;
6307
6308        perf_sample_data_init(&sample, bp->attr.bp_addr, 0);
6309
6310        if (!bp->hw.state && !perf_exclude_event(bp, regs))
6311                perf_swevent_event(bp, 1, &sample, regs);
6312}
6313#endif
6314
6315/*
6316 * hrtimer based swevent callback
6317 */
6318
6319static enum hrtimer_restart perf_swevent_hrtimer(struct hrtimer *hrtimer)
6320{
6321        enum hrtimer_restart ret = HRTIMER_RESTART;
6322        struct perf_sample_data data;
6323        struct pt_regs *regs;
6324        struct perf_event *event;
6325        u64 period;
6326
6327        event = container_of(hrtimer, struct perf_event, hw.hrtimer);
6328
6329        if (event->state != PERF_EVENT_STATE_ACTIVE)
6330                return HRTIMER_NORESTART;
6331
6332        event->pmu->read(event);
6333
6334        perf_sample_data_init(&data, 0, event->hw.last_period);
6335        regs = get_irq_regs();
6336
6337        if (regs && !perf_exclude_event(event, regs)) {
6338                if (!(event->attr.exclude_idle && is_idle_task(current)))
6339                        if (__perf_event_overflow(event, 1, &data, regs))
6340                                ret = HRTIMER_NORESTART;
6341        }
6342
6343        period = max_t(u64, 10000, event->hw.sample_period);
6344        hrtimer_forward_now(hrtimer, ns_to_ktime(period));
6345
6346        return ret;
6347}
6348
6349static void perf_swevent_start_hrtimer(struct perf_event *event)
6350{
6351        struct hw_perf_event *hwc = &event->hw;
6352        s64 period;
6353
6354        if (!is_sampling_event(event))
6355                return;
6356
6357        period = local64_read(&hwc->period_left);
6358        if (period) {
6359                if (period < 0)
6360                        period = 10000;
6361
6362                local64_set(&hwc->period_left, 0);
6363        } else {
6364                period = max_t(u64, 10000, hwc->sample_period);
6365        }
6366        __hrtimer_start_range_ns(&hwc->hrtimer,
6367                                ns_to_ktime(period), 0,
6368                                HRTIMER_MODE_REL_PINNED, 0);
6369}
6370
6371static void perf_swevent_cancel_hrtimer(struct perf_event *event)
6372{
6373        struct hw_perf_event *hwc = &event->hw;
6374
6375        if (is_sampling_event(event)) {
6376                ktime_t remaining = hrtimer_get_remaining(&hwc->hrtimer);
6377                local64_set(&hwc->period_left, ktime_to_ns(remaining));
6378
6379                hrtimer_cancel(&hwc->hrtimer);
6380        }
6381}
6382
6383static void perf_swevent_init_hrtimer(struct perf_event *event)
6384{
6385        struct hw_perf_event *hwc = &event->hw;
6386
6387        if (!is_sampling_event(event))
6388                return;
6389
6390        hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
6391        hwc->hrtimer.function = perf_swevent_hrtimer;
6392
6393        /*
6394         * Since hrtimers have a fixed rate, we can do a static freq->period
6395         * mapping and avoid the whole period adjust feedback stuff.
6396         */
6397        if (event->attr.freq) {
6398                long freq = event->attr.sample_freq;
6399
6400                event->attr.sample_period = NSEC_PER_SEC / freq;
6401                hwc->sample_period = event->attr.sample_period;
6402                local64_set(&hwc->period_left, hwc->sample_period);
6403                hwc->last_period = hwc->sample_period;
6404                event->attr.freq = 0;
6405        }
6406}
6407
6408/*
6409 * Software event: cpu wall time clock
6410 */
6411
6412static void cpu_clock_event_update(struct perf_event *event)
6413{
6414        s64 prev;
6415        u64 now;
6416
6417        now = local_clock();
6418        prev = local64_xchg(&event->hw.prev_count, now);
6419        local64_add(now - prev, &event->count);
6420}
6421
6422static void cpu_clock_event_start(struct perf_event *event, int flags)
6423{
6424        local64_set(&event->hw.prev_count, local_clock());
6425        perf_swevent_start_hrtimer(event);
6426}
6427
6428static void cpu_clock_event_stop(struct perf_event *event, int flags)
6429{
6430        perf_swevent_cancel_hrtimer(event);
6431        cpu_clock_event_update(event);
6432}
6433
6434static int cpu_clock_event_add(struct perf_event *event, int flags)
6435{
6436        if (flags & PERF_EF_START)
6437                cpu_clock_event_start(event, flags);
6438
6439        return 0;
6440}
6441
6442static void cpu_clock_event_del(struct perf_event *event, int flags)
6443{
6444        cpu_clock_event_stop(event, flags);
6445}
6446
6447static void cpu_clock_event_read(struct perf_event *event)
6448{
6449        cpu_clock_event_update(event);
6450}
6451
6452static int cpu_clock_event_init(struct perf_event *event)
6453{
6454        if (event->attr.type != PERF_TYPE_SOFTWARE)
6455                return -ENOENT;
6456
6457        if (event->attr.config != PERF_COUNT_SW_CPU_CLOCK)
6458                return -ENOENT;
6459
6460        /*
6461         * no branch sampling for software events
6462         */
6463        if (has_branch_stack(event))
6464                return -EOPNOTSUPP;
6465
6466        perf_swevent_init_hrtimer(event);
6467
6468        return 0;
6469}
6470
6471static struct pmu perf_cpu_clock = {
6472        .task_ctx_nr    = perf_sw_context,
6473
6474        .event_init     = cpu_clock_event_init,
6475        .add            = cpu_clock_event_add,
6476        .del            = cpu_clock_event_del,
6477        .start          = cpu_clock_event_start,
6478        .stop           = cpu_clock_event_stop,
6479        .read           = cpu_clock_event_read,
6480};
6481
6482/*
6483 * Software event: task time clock
6484 */
6485
6486static void task_clock_event_update(struct perf_event *event, u64 now)
6487{
6488        u64 prev;
6489        s64 delta;
6490
6491        prev = local64_xchg(&event->hw.prev_count, now);
6492        delta = now - prev;
6493        local64_add(delta, &event->count);
6494}
6495
6496static void task_clock_event_start(struct perf_event *event, int flags)
6497{
6498        local64_set(&event->hw.prev_count, event->ctx->time);
6499        perf_swevent_start_hrtimer(event);
6500}
6501
6502static void task_clock_event_stop(struct perf_event *event, int flags)
6503{
6504        perf_swevent_cancel_hrtimer(event);
6505        task_clock_event_update(event, event->ctx->time);
6506}
6507
6508static int task_clock_event_add(struct perf_event *event, int flags)
6509{
6510        if (flags & PERF_EF_START)
6511                task_clock_event_start(event, flags);
6512
6513        return 0;
6514}
6515
6516static void task_clock_event_del(struct perf_event *event, int flags)
6517{
6518        task_clock_event_stop(event, PERF_EF_UPDATE);
6519}
6520
6521static void task_clock_event_read(struct perf_event *event)
6522{
6523        u64 now = perf_clock();
6524        u64 delta = now - event->ctx->timestamp;
6525        u64 time = event->ctx->time + delta;
6526
6527        task_clock_event_update(event, time);
6528}
6529
6530static int task_clock_event_init(struct perf_event *event)
6531{
6532        if (event->attr.type != PERF_TYPE_SOFTWARE)
6533                return -ENOENT;
6534
6535        if (event->attr.config != PERF_COUNT_SW_TASK_CLOCK)
6536                return -ENOENT;
6537
6538        /*
6539         * no branch sampling for software events
6540         */
6541        if (has_branch_stack(event))
6542                return -EOPNOTSUPP;
6543
6544        perf_swevent_init_hrtimer(event);
6545
6546        return 0;
6547}
6548
6549static struct pmu perf_task_clock = {
6550        .task_ctx_nr    = perf_sw_context,
6551
6552        .event_init     = task_clock_event_init,
6553        .add            = task_clock_event_add,
6554        .del            = task_clock_event_del,
6555        .start          = task_clock_event_start,
6556        .stop           = task_clock_event_stop,
6557        .read           = task_clock_event_read,
6558};
6559
6560static void perf_pmu_nop_void(struct pmu *pmu)
6561{
6562}
6563
6564static int perf_pmu_nop_int(struct pmu *pmu)
6565{
6566        return 0;
6567}
6568
6569static void perf_pmu_start_txn(struct pmu *pmu)
6570{
6571        perf_pmu_disable(pmu);
6572}
6573
6574static int perf_pmu_commit_txn(struct pmu *pmu)
6575{
6576        perf_pmu_enable(pmu);
6577        return 0;
6578}
6579
6580static void perf_pmu_cancel_txn(struct pmu *pmu)
6581{
6582        perf_pmu_enable(pmu);
6583}
6584
6585static int perf_event_idx_default(struct perf_event *event)
6586{
6587        return 0;
6588}
6589
6590/*
6591 * Ensures all contexts with the same task_ctx_nr have the same
6592 * pmu_cpu_context too.
6593 */
6594static struct perf_cpu_context __percpu *find_pmu_context(int ctxn)
6595{
6596        struct pmu *pmu;
6597
6598        if (ctxn < 0)
6599                return NULL;
6600
6601        list_for_each_entry(pmu, &pmus, entry) {
6602                if (pmu->task_ctx_nr == ctxn)
6603                        return pmu->pmu_cpu_context;
6604        }
6605
6606        return NULL;
6607}
6608
6609static void update_pmu_context(struct pmu *pmu, struct pmu *old_pmu)
6610{
6611        int cpu;
6612
6613        for_each_possible_cpu(cpu) {
6614                struct perf_cpu_context *cpuctx;
6615
6616                cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6617
6618                if (cpuctx->unique_pmu == old_pmu)
6619                        cpuctx->unique_pmu = pmu;
6620        }
6621}
6622
6623static void free_pmu_context(struct pmu *pmu)
6624{
6625        struct pmu *i;
6626
6627        mutex_lock(&pmus_lock);
6628        /*
6629         * Like a real lame refcount.
6630         */
6631        list_for_each_entry(i, &pmus, entry) {
6632                if (i->pmu_cpu_context == pmu->pmu_cpu_context) {
6633                        update_pmu_context(i, pmu);
6634                        goto out;
6635                }
6636        }
6637
6638        free_percpu(pmu->pmu_cpu_context);
6639out:
6640        mutex_unlock(&pmus_lock);
6641}
6642static struct idr pmu_idr;
6643
6644static ssize_t
6645type_show(struct device *dev, struct device_attribute *attr, char *page)
6646{
6647        struct pmu *pmu = dev_get_drvdata(dev);
6648
6649        return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->type);
6650}
6651static DEVICE_ATTR_RO(type);
6652
6653static ssize_t
6654perf_event_mux_interval_ms_show(struct device *dev,
6655                                struct device_attribute *attr,
6656                                char *page)
6657{
6658        struct pmu *pmu = dev_get_drvdata(dev);
6659
6660        return snprintf(page, PAGE_SIZE-1, "%d\n", pmu->hrtimer_interval_ms);
6661}
6662
6663static ssize_t
6664perf_event_mux_interval_ms_store(struct device *dev,
6665                                 struct device_attribute *attr,
6666                                 const char *buf, size_t count)
6667{
6668        struct pmu *pmu = dev_get_drvdata(dev);
6669        int timer, cpu, ret;
6670
6671        ret = kstrtoint(buf, 0, &timer);
6672        if (ret)
6673                return ret;
6674
6675        if (timer < 1)
6676                return -EINVAL;
6677
6678        /* same value, noting to do */
6679        if (timer == pmu->hrtimer_interval_ms)
6680                return count;
6681
6682        pmu->hrtimer_interval_ms = timer;
6683
6684        /* update all cpuctx for this PMU */
6685        for_each_possible_cpu(cpu) {
6686                struct perf_cpu_context *cpuctx;
6687                cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6688                cpuctx->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);
6689
6690                if (hrtimer_active(&cpuctx->hrtimer))
6691                        hrtimer_forward_now(&cpuctx->hrtimer, cpuctx->hrtimer_interval);
6692        }
6693
6694        return count;
6695}
6696static DEVICE_ATTR_RW(perf_event_mux_interval_ms);
6697
6698static struct attribute *pmu_dev_attrs[] = {
6699        &dev_attr_type.attr,
6700        &dev_attr_perf_event_mux_interval_ms.attr,
6701        NULL,
6702};
6703ATTRIBUTE_GROUPS(pmu_dev);
6704
6705static int pmu_bus_running;
6706static struct bus_type pmu_bus = {
6707        .name           = "event_source",
6708        .dev_groups     = pmu_dev_groups,
6709};
6710
6711static void pmu_dev_release(struct device *dev)
6712{
6713        kfree(dev);
6714}
6715
6716static int pmu_dev_alloc(struct pmu *pmu)
6717{
6718        int ret = -ENOMEM;
6719
6720        pmu->dev = kzalloc(sizeof(struct device), GFP_KERNEL);
6721        if (!pmu->dev)
6722                goto out;
6723
6724        pmu->dev->groups = pmu->attr_groups;
6725        device_initialize(pmu->dev);
6726        ret = dev_set_name(pmu->dev, "%s", pmu->name);
6727        if (ret)
6728                goto free_dev;
6729
6730        dev_set_drvdata(pmu->dev, pmu);
6731        pmu->dev->bus = &pmu_bus;
6732        pmu->dev->release = pmu_dev_release;
6733        ret = device_add(pmu->dev);
6734        if (ret)
6735                goto free_dev;
6736
6737out:
6738        return ret;
6739
6740free_dev:
6741        put_device(pmu->dev);
6742        goto out;
6743}
6744
6745static struct lock_class_key cpuctx_mutex;
6746static struct lock_class_key cpuctx_lock;
6747
6748int perf_pmu_register(struct pmu *pmu, const char *name, int type)
6749{
6750        int cpu, ret;
6751
6752        mutex_lock(&pmus_lock);
6753        ret = -ENOMEM;
6754        pmu->pmu_disable_count = alloc_percpu(int);
6755        if (!pmu->pmu_disable_count)
6756                goto unlock;
6757
6758        pmu->type = -1;
6759        if (!name)
6760                goto skip_type;
6761        pmu->name = name;
6762
6763        if (type < 0) {
6764                type = idr_alloc(&pmu_idr, pmu, PERF_TYPE_MAX, 0, GFP_KERNEL);
6765                if (type < 0) {
6766                        ret = type;
6767                        goto free_pdc;
6768                }
6769        }
6770        pmu->type = type;
6771
6772        if (pmu_bus_running) {
6773                ret = pmu_dev_alloc(pmu);
6774                if (ret)
6775                        goto free_idr;
6776        }
6777
6778skip_type:
6779        pmu->pmu_cpu_context = find_pmu_context(pmu->task_ctx_nr);
6780        if (pmu->pmu_cpu_context)
6781                goto got_cpu_context;
6782
6783        ret = -ENOMEM;
6784        pmu->pmu_cpu_context = alloc_percpu(struct perf_cpu_context);
6785        if (!pmu->pmu_cpu_context)
6786                goto free_dev;
6787
6788        for_each_possible_cpu(cpu) {
6789                struct perf_cpu_context *cpuctx;
6790
6791                cpuctx = per_cpu_ptr(pmu->pmu_cpu_context, cpu);
6792                __perf_event_init_context(&cpuctx->ctx);
6793                lockdep_set_class(&cpuctx->ctx.mutex, &cpuctx_mutex);
6794                lockdep_set_class(&cpuctx->ctx.lock, &cpuctx_lock);
6795                cpuctx->ctx.type = cpu_context;
6796                cpuctx->ctx.pmu = pmu;
6797
6798                __perf_cpu_hrtimer_init(cpuctx, cpu);
6799
6800                INIT_LIST_HEAD(&cpuctx->rotation_list);
6801                cpuctx->unique_pmu = pmu;
6802        }
6803
6804got_cpu_context:
6805        if (!pmu->start_txn) {
6806                if (pmu->pmu_enable) {
6807                        /*
6808                         * If we have pmu_enable/pmu_disable calls, install
6809                         * transaction stubs that use that to try and batch
6810                         * hardware accesses.
6811                         */
6812                        pmu->start_txn  = perf_pmu_start_txn;
6813                        pmu->commit_txn = perf_pmu_commit_txn;
6814                        pmu->cancel_txn = perf_pmu_cancel_txn;
6815                } else {
6816                        pmu->start_txn  = perf_pmu_nop_void;
6817                        pmu->commit_txn = perf_pmu_nop_int;
6818                        pmu->cancel_txn = perf_pmu_nop_void;
6819                }
6820        }
6821
6822        if (!pmu->pmu_enable) {
6823                pmu->pmu_enable  = perf_pmu_nop_void;
6824                pmu->pmu_disable = perf_pmu_nop_void;
6825        }
6826
6827        if (!pmu->event_idx)
6828                pmu->event_idx = perf_event_idx_default;
6829
6830        list_add_rcu(&pmu->entry, &pmus);
6831        ret = 0;
6832unlock:
6833        mutex_unlock(&pmus_lock);
6834
6835        return ret;
6836
6837free_dev:
6838        device_del(pmu->dev);
6839        put_device(pmu->dev);
6840
6841free_idr:
6842        if (pmu->type >= PERF_TYPE_MAX)
6843                idr_remove(&pmu_idr, pmu->type);
6844
6845free_pdc:
6846        free_percpu(pmu->pmu_disable_count);
6847        goto unlock;
6848}
6849EXPORT_SYMBOL_GPL(perf_pmu_register);
6850
6851void perf_pmu_unregister(struct pmu *pmu)
6852{
6853        mutex_lock(&pmus_lock);
6854        list_del_rcu(&pmu->entry);
6855        mutex_unlock(&pmus_lock);
6856
6857        /*
6858         * We dereference the pmu list under both SRCU and regular RCU, so
6859         * synchronize against both of those.
6860         */
6861        synchronize_srcu(&pmus_srcu);
6862        synchronize_rcu();
6863
6864        free_percpu(pmu->pmu_disable_count);
6865        if (pmu->type >= PERF_TYPE_MAX)
6866                idr_remove(&pmu_idr, pmu->type);
6867        device_del(pmu->dev);
6868        put_device(pmu->dev);
6869        free_pmu_context(pmu);
6870}
6871EXPORT_SYMBOL_GPL(perf_pmu_unregister);
6872
6873struct pmu *perf_init_event(struct perf_event *event)
6874{
6875        struct pmu *pmu = NULL;
6876        int idx;
6877        int ret;
6878
6879        idx = srcu_read_lock(&pmus_srcu);
6880
6881        rcu_read_lock();
6882        pmu = idr_find(&pmu_idr, event->attr.type);
6883        rcu_read_unlock();
6884        if (pmu) {
6885                if (!try_module_get(pmu->module)) {
6886                        pmu = ERR_PTR(-ENODEV);
6887                        goto unlock;
6888                }
6889                event->pmu = pmu;
6890                ret = pmu->event_init(event);
6891                if (ret)
6892                        pmu = ERR_PTR(ret);
6893                goto unlock;
6894        }
6895
6896        list_for_each_entry_rcu(pmu, &pmus, entry) {
6897                if (!try_module_get(pmu->module)) {
6898                        pmu = ERR_PTR(-ENODEV);
6899                        goto unlock;
6900                }
6901                event->pmu = pmu;
6902                ret = pmu->event_init(event);
6903                if (!ret)
6904                        goto unlock;
6905
6906                if (ret != -ENOENT) {
6907                        pmu = ERR_PTR(ret);
6908                        goto unlock;
6909                }
6910        }
6911        pmu = ERR_PTR(-ENOENT);
6912unlock:
6913        srcu_read_unlock(&pmus_srcu, idx);
6914
6915        return pmu;
6916}
6917
6918static void account_event_cpu(struct perf_event *event, int cpu)
6919{
6920        if (event->parent)
6921                return;
6922
6923        if (has_branch_stack(event)) {
6924                if (!(event->attach_state & PERF_ATTACH_TASK))
6925                        atomic_inc(&per_cpu(perf_branch_stack_events, cpu));
6926        }
6927        if (is_cgroup_event(event))
6928                atomic_inc(&per_cpu(perf_cgroup_events, cpu));
6929}
6930
6931static void account_event(struct perf_event *event)
6932{
6933        if (event->parent)
6934                return;
6935
6936        if (event->attach_state & PERF_ATTACH_TASK)
6937                static_key_slow_inc(&perf_sched_events.key);
6938        if (event->attr.mmap || event->attr.mmap_data)
6939                atomic_inc(&nr_mmap_events);
6940        if (event->attr.comm)
6941                atomic_inc(&nr_comm_events);
6942        if (event->attr.task)
6943                atomic_inc(&nr_task_events);
6944        if (event->attr.freq) {
6945                if (atomic_inc_return(&nr_freq_events) == 1)
6946                        tick_nohz_full_kick_all();
6947        }
6948        if (has_branch_stack(event))
6949                static_key_slow_inc(&perf_sched_events.key);
6950        if (is_cgroup_event(event))
6951                static_key_slow_inc(&perf_sched_events.key);
6952
6953        account_event_cpu(event, event->cpu);
6954}
6955
6956/*
6957 * Allocate and initialize a event structure
6958 */
6959static struct perf_event *
6960perf_event_alloc(struct perf_event_attr *attr, int cpu,
6961                 struct task_struct *task,
6962                 struct perf_event *group_leader,
6963                 struct perf_event *parent_event,
6964                 perf_overflow_handler_t overflow_handler,
6965                 void *context)
6966{
6967        struct pmu *pmu;
6968        struct perf_event *event;
6969        struct hw_perf_event *hwc;
6970        long err = -EINVAL;
6971
6972        if ((unsigned)cpu >= nr_cpu_ids) {
6973                if (!task || cpu != -1)
6974                        return ERR_PTR(-EINVAL);
6975        }
6976
6977        event = kzalloc(sizeof(*event), GFP_KERNEL);
6978        if (!event)
6979                return ERR_PTR(-ENOMEM);
6980
6981        /*
6982         * Single events are their own group leaders, with an
6983         * empty sibling list:
6984         */
6985        if (!group_leader)
6986                group_leader = event;
6987
6988        mutex_init(&event->child_mutex);
6989        INIT_LIST_HEAD(&event->child_list);
6990
6991        INIT_LIST_HEAD(&event->group_entry);
6992        INIT_LIST_HEAD(&event->event_entry);
6993        INIT_LIST_HEAD(&event->sibling_list);
6994        INIT_LIST_HEAD(&event->rb_entry);
6995        INIT_LIST_HEAD(&event->active_entry);
6996        INIT_HLIST_NODE(&event->hlist_entry);
6997
6998
6999        init_waitqueue_head(&event->waitq);
7000        init_irq_work(&event->pending, perf_pending_event);
7001
7002        mutex_init(&event->mmap_mutex);
7003
7004        atomic_long_set(&event->refcount, 1);
7005        event->cpu              = cpu;
7006        event->attr             = *attr;
7007        event->group_leader     = group_leader;
7008        event->pmu              = NULL;
7009        event->oncpu            = -1;
7010
7011        event->parent           = parent_event;
7012
7013        event->ns               = get_pid_ns(task_active_pid_ns(current));
7014        event->id               = atomic64_inc_return(&perf_event_id);
7015
7016        event->state            = PERF_EVENT_STATE_INACTIVE;
7017
7018        if (task) {
7019                event->attach_state = PERF_ATTACH_TASK;
7020
7021                if (attr->type == PERF_TYPE_TRACEPOINT)
7022                        event->hw.tp_target = task;
7023#ifdef CONFIG_HAVE_HW_BREAKPOINT
7024                /*
7025                 * hw_breakpoint is a bit difficult here..
7026                 */
7027                else if (attr->type == PERF_TYPE_BREAKPOINT)
7028                        event->hw.bp_target = task;
7029#endif
7030        }
7031
7032        if (!overflow_handler && parent_event) {
7033                overflow_handler = parent_event->overflow_handler;
7034                context = parent_event->overflow_handler_context;
7035        }
7036
7037        event->overflow_handler = overflow_handler;
7038        event->overflow_handler_context = context;
7039
7040        perf_event__state_init(event);
7041
7042        pmu = NULL;
7043
7044        hwc = &event->hw;
7045        hwc->sample_period = attr->sample_period;
7046        if (attr->freq && attr->sample_freq)
7047                hwc->sample_period = 1;
7048        hwc->last_period = hwc->sample_period;
7049
7050        local64_set(&hwc->period_left, hwc->sample_period);
7051
7052        /*
7053         * we currently do not support PERF_FORMAT_GROUP on inherited events
7054         */
7055        if (attr->inherit && (attr->read_format & PERF_FORMAT_GROUP))
7056                goto err_ns;
7057
7058        pmu = perf_init_event(event);
7059        if (!pmu)
7060                goto err_ns;
7061        else if (IS_ERR(pmu)) {
7062                err = PTR_ERR(pmu);
7063                goto err_ns;
7064        }
7065
7066        if (!event->parent) {
7067                if (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) {
7068                        err = get_callchain_buffers();
7069                        if (err)
7070                                goto err_pmu;
7071                }
7072        }
7073
7074        return event;
7075
7076err_pmu:
7077        if (event->destroy)
7078                event->destroy(event);
7079        module_put(pmu->module);
7080err_ns:
7081        if (event->ns)
7082                put_pid_ns(event->ns);
7083        kfree(event);
7084
7085        return ERR_PTR(err);
7086}
7087
7088static int perf_copy_attr(struct perf_event_attr __user *uattr,
7089                          struct perf_event_attr *attr)
7090{
7091        u32 size;
7092        int ret;
7093
7094        if (!access_ok(VERIFY_WRITE, uattr, PERF_ATTR_SIZE_VER0))
7095                return -EFAULT;
7096
7097        /*
7098         * zero the full structure, so that a short copy will be nice.
7099         */
7100        memset(attr, 0, sizeof(*attr));
7101
7102        ret = get_user(size, &uattr->size);
7103        if (ret)
7104                return ret;
7105
7106        if (size > PAGE_SIZE)   /* silly large */
7107                goto err_size;
7108
7109        if (!size)              /* abi compat */
7110                size = PERF_ATTR_SIZE_VER0;
7111
7112        if (size < PERF_ATTR_SIZE_VER0)
7113                goto err_size;
7114
7115        /*
7116         * If we're handed a bigger struct than we know of,
7117         * ensure all the unknown bits are 0 - i.e. new
7118         * user-space does not rely on any kernel feature
7119         * extensions we dont know about yet.
7120         */
7121        if (size > sizeof(*attr)) {
7122                unsigned char __user *addr;
7123                unsigned char __user *end;
7124                unsigned char val;
7125
7126                addr = (void __user *)uattr + sizeof(*attr);
7127                end  = (void __user *)uattr + size;
7128
7129                for (; addr < end; addr++) {
7130                        ret = get_user(val, addr);
7131                        if (ret)
7132                                return ret;
7133                        if (val)
7134                                goto err_size;
7135                }
7136                size = sizeof(*attr);
7137        }
7138
7139        ret = copy_from_user(attr, uattr, size);
7140        if (ret)
7141                return -EFAULT;
7142
7143        if (attr->__reserved_1)
7144                return -EINVAL;
7145
7146        if (attr->sample_type & ~(PERF_SAMPLE_MAX-1))
7147                return -EINVAL;
7148
7149        if (attr->read_format & ~(PERF_FORMAT_MAX-1))
7150                return -EINVAL;
7151
7152        if (attr->sample_type & PERF_SAMPLE_BRANCH_STACK) {
7153                u64 mask = attr->branch_sample_type;
7154
7155                /* only using defined bits */
7156                if (mask & ~(PERF_SAMPLE_BRANCH_MAX-1))
7157                        return -EINVAL;
7158
7159                /* at least one branch bit must be set */
7160                if (!(mask & ~PERF_SAMPLE_BRANCH_PLM_ALL))
7161                        return -EINVAL;
7162
7163                /* propagate priv level, when not set for branch */
7164                if (!(mask & PERF_SAMPLE_BRANCH_PLM_ALL)) {
7165
7166                        /* exclude_kernel checked on syscall entry */
7167                        if (!attr->exclude_kernel)
7168                                mask |= PERF_SAMPLE_BRANCH_KERNEL;
7169
7170                        if (!attr->exclude_user)
7171                                mask |= PERF_SAMPLE_BRANCH_USER;
7172
7173                        if (!attr->exclude_hv)
7174                                mask |= PERF_SAMPLE_BRANCH_HV;
7175                        /*
7176                         * adjust user setting (for HW filter setup)
7177                         */
7178                        attr->branch_sample_type = mask;
7179                }
7180                /* privileged levels capture (kernel, hv): check permissions */
7181                if ((mask & PERF_SAMPLE_BRANCH_PERM_PLM)
7182                    && perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
7183                        return -EACCES;
7184        }
7185
7186        if (attr->sample_type & PERF_SAMPLE_REGS_USER) {
7187                ret = perf_reg_validate(attr->sample_regs_user);
7188                if (ret)
7189                        return ret;
7190        }
7191
7192        if (attr->sample_type & PERF_SAMPLE_STACK_USER) {
7193                if (!arch_perf_have_user_stack_dump())
7194                        return -ENOSYS;
7195
7196                /*
7197                 * We have __u32 type for the size, but so far
7198                 * we can only use __u16 as maximum due to the
7199                 * __u16 sample size limit.
7200                 */
7201                if (attr->sample_stack_user >= USHRT_MAX)
7202                        ret = -EINVAL;
7203                else if (!IS_ALIGNED(attr->sample_stack_user, sizeof(u64)))
7204                        ret = -EINVAL;
7205        }
7206
7207out:
7208        return ret;
7209
7210err_size:
7211        put_user(sizeof(*attr), &uattr->size);
7212        ret = -E2BIG;
7213        goto out;
7214}
7215
7216static int
7217perf_event_set_output(struct perf_event *event, struct perf_event *output_event)
7218{
7219        struct ring_buffer *rb = NULL;
7220        int ret = -EINVAL;
7221
7222        if (!output_event)
7223                goto set;
7224
7225        /* don't allow circular references */
7226        if (event == output_event)
7227                goto out;
7228
7229        /*
7230         * Don't allow cross-cpu buffers
7231         */
7232        if (output_event->cpu != event->cpu)
7233                goto out;
7234
7235        /*
7236         * If its not a per-cpu rb, it must be the same task.
7237         */
7238        if (output_event->cpu == -1 && output_event->ctx != event->ctx)
7239                goto out;
7240
7241set:
7242        mutex_lock(&event->mmap_mutex);
7243        /* Can't redirect output if we've got an active mmap() */
7244        if (atomic_read(&event->mmap_count))
7245                goto unlock;
7246
7247        if (output_event) {
7248                /* get the rb we want to redirect to */
7249                rb = ring_buffer_get(output_event);
7250                if (!rb)
7251                        goto unlock;
7252        }
7253
7254        ring_buffer_attach(event, rb);
7255
7256        ret = 0;
7257unlock:
7258        mutex_unlock(&event->mmap_mutex);
7259
7260out:
7261        return ret;
7262}
7263
7264/**
7265 * sys_perf_event_open - open a performance event, associate it to a task/cpu
7266 *
7267 * @attr_uptr:  event_id type attributes for monitoring/sampling
7268 * @pid:                target pid
7269 * @cpu:                target cpu
7270 * @group_fd:           group leader event fd
7271 */
7272SYSCALL_DEFINE5(perf_event_open,
7273                struct perf_event_attr __user *, attr_uptr,
7274                pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
7275{
7276        struct perf_event *group_leader = NULL, *output_event = NULL;
7277        struct perf_event *event, *sibling;
7278        struct perf_event_attr attr;
7279        struct perf_event_context *ctx;
7280        struct file *event_file = NULL;
7281        struct fd group = {NULL, 0};
7282        struct task_struct *task = NULL;
7283        struct pmu *pmu;
7284        int event_fd;
7285        int move_group = 0;
7286        int err;
7287        int f_flags = O_RDWR;
7288
7289        /* for future expandability... */
7290        if (flags & ~PERF_FLAG_ALL)
7291                return -EINVAL;
7292
7293        err = perf_copy_attr(attr_uptr, &attr);
7294        if (err)
7295                return err;
7296
7297        if (!attr.exclude_kernel) {
7298                if (perf_paranoid_kernel() && !capable(CAP_SYS_ADMIN))
7299                        return -EACCES;
7300        }
7301
7302        if (attr.freq) {
7303                if (attr.sample_freq > sysctl_perf_event_sample_rate)
7304                        return -EINVAL;
7305        } else {
7306                if (attr.sample_period & (1ULL << 63))
7307                        return -EINVAL;
7308        }
7309
7310        /*
7311         * In cgroup mode, the pid argument is used to pass the fd
7312         * opened to the cgroup directory in cgroupfs. The cpu argument
7313         * designates the cpu on which to monitor threads from that
7314         * cgroup.
7315         */
7316        if ((flags & PERF_FLAG_PID_CGROUP) && (pid == -1 || cpu == -1))
7317                return -EINVAL;
7318
7319        if (flags & PERF_FLAG_FD_CLOEXEC)
7320                f_flags |= O_CLOEXEC;
7321
7322        event_fd = get_unused_fd_flags(f_flags);
7323        if (event_fd < 0)
7324                return event_fd;
7325
7326        if (group_fd != -1) {
7327                err = perf_fget_light(group_fd, &group);
7328                if (err)
7329                        goto err_fd;
7330                group_leader = group.file->private_data;
7331                if (flags & PERF_FLAG_FD_OUTPUT)
7332                        output_event = group_leader;
7333                if (flags & PERF_FLAG_FD_NO_GROUP)
7334                        group_leader = NULL;
7335        }
7336
7337        if (pid != -1 && !(flags & PERF_FLAG_PID_CGROUP)) {
7338                task = find_lively_task_by_vpid(pid);
7339                if (IS_ERR(task)) {
7340                        err = PTR_ERR(task);
7341                        goto err_group_fd;
7342                }
7343        }
7344
7345        if (task && group_leader &&
7346            group_leader->attr.inherit != attr.inherit) {
7347                err = -EINVAL;
7348                goto err_task;
7349        }
7350
7351        get_online_cpus();
7352
7353        event = perf_event_alloc(&attr, cpu, task, group_leader, NULL,
7354                                 NULL, NULL);
7355        if (IS_ERR(event)) {
7356                err = PTR_ERR(event);
7357                goto err_cpus;
7358        }
7359
7360        if (flags & PERF_FLAG_PID_CGROUP) {
7361                err = perf_cgroup_connect(pid, event, &attr, group_leader);
7362                if (err) {
7363                        __free_event(event);
7364                        goto err_cpus;
7365                }
7366        }
7367
7368        if (is_sampling_event(event)) {
7369                if (event->pmu->capabilities & PERF_PMU_CAP_NO_INTERRUPT) {
7370                        err = -ENOTSUPP;
7371                        goto err_alloc;
7372                }
7373        }
7374
7375        account_event(event);
7376
7377        /*
7378         * Special case software events and allow them to be part of
7379         * any hardware group.
7380         */
7381        pmu = event->pmu;
7382
7383        if (group_leader &&
7384            (is_software_event(event) != is_software_event(group_leader))) {
7385                if (is_software_event(event)) {
7386                        /*
7387                         * If event and group_leader are not both a software
7388                         * event, and event is, then group leader is not.
7389                         *
7390                         * Allow the addition of software events to !software
7391                         * groups, this is safe because software events never
7392                         * fail to schedule.
7393                         */
7394                        pmu = group_leader->pmu;
7395                } else if (is_software_event(group_leader) &&
7396                           (group_leader->group_flags & PERF_GROUP_SOFTWARE)) {
7397                        /*
7398                         * In case the group is a pure software group, and we
7399                         * try to add a hardware event, move the whole group to
7400                         * the hardware context.
7401                         */
7402                        move_group = 1;
7403                }
7404        }
7405
7406        /*
7407         * Get the target context (task or percpu):
7408         */
7409        ctx = find_get_context(pmu, task, event->cpu);
7410        if (IS_ERR(ctx)) {
7411                err = PTR_ERR(ctx);
7412                goto err_alloc;
7413        }
7414
7415        if (task) {
7416                put_task_struct(task);
7417                task = NULL;
7418        }
7419
7420        /*
7421         * Look up the group leader (we will attach this event to it):
7422         */
7423        if (group_leader) {
7424                err = -EINVAL;
7425
7426                /*
7427                 * Do not allow a recursive hierarchy (this new sibling
7428                 * becoming part of another group-sibling):
7429                 */
7430                if (group_leader->group_leader != group_leader)
7431                        goto err_context;
7432                /*
7433                 * Do not allow to attach to a group in a different
7434                 * task or CPU context:
7435                 */
7436                if (move_group) {
7437                        if (group_leader->ctx->type != ctx->type)
7438                                goto err_context;
7439                } else {
7440                        if (group_leader->ctx != ctx)
7441                                goto err_context;
7442                }
7443
7444                /*
7445                 * Only a group leader can be exclusive or pinned
7446                 */
7447                if (attr.exclusive || attr.pinned)
7448                        goto err_context;
7449        }
7450
7451        if (output_event) {
7452                err = perf_event_set_output(event, output_event);
7453                if (err)
7454                        goto err_context;
7455        }
7456
7457        event_file = anon_inode_getfile("[perf_event]", &perf_fops, event,
7458                                        f_flags);
7459        if (IS_ERR(event_file)) {
7460                err = PTR_ERR(event_file);
7461                goto err_context;
7462        }
7463
7464        if (move_group) {
7465                struct perf_event_context *gctx = group_leader->ctx;
7466
7467                mutex_lock(&gctx->mutex);
7468                perf_remove_from_context(group_leader, false);
7469
7470                /*
7471                 * Removing from the context ends up with disabled
7472                 * event. What we want here is event in the initial
7473                 * startup state, ready to be add into new context.
7474                 */
7475                perf_event__state_init(group_leader);
7476                list_for_each_entry(sibling, &group_leader->sibling_list,
7477                                    group_entry) {
7478                        perf_remove_from_context(sibling, false);
7479                        perf_event__state_init(sibling);
7480                        put_ctx(gctx);
7481                }
7482                mutex_unlock(&gctx->mutex);
7483                put_ctx(gctx);
7484        }
7485
7486        WARN_ON_ONCE(ctx->parent_ctx);
7487        mutex_lock(&ctx->mutex);
7488
7489        if (move_group) {
7490                synchronize_rcu();
7491                perf_install_in_context(ctx, group_leader, group_leader->cpu);
7492                get_ctx(ctx);
7493                list_for_each_entry(sibling, &group_leader->sibling_list,
7494                                    group_entry) {
7495                        perf_install_in_context(ctx, sibling, sibling->cpu);
7496                        get_ctx(ctx);
7497                }
7498        }
7499
7500        perf_install_in_context(ctx, event, event->cpu);
7501        perf_unpin_context(ctx);
7502        mutex_unlock(&ctx->mutex);
7503
7504        put_online_cpus();
7505
7506        event->owner = current;
7507
7508        mutex_lock(&current->perf_event_mutex);
7509        list_add_tail(&event->owner_entry, &current->perf_event_list);
7510        mutex_unlock(&current->perf_event_mutex);
7511
7512        /*
7513         * Precalculate sample_data sizes
7514         */
7515        perf_event__header_size(event);
7516        perf_event__id_header_size(event);
7517
7518        /*
7519         * Drop the reference on the group_event after placing the
7520         * new event on the sibling_list. This ensures destruction
7521         * of the group leader will find the pointer to itself in
7522         * perf_group_detach().
7523         */
7524        fdput(group);
7525        fd_install(event_fd, event_file);
7526        return event_fd;
7527
7528err_context:
7529        perf_unpin_context(ctx);
7530        put_ctx(ctx);
7531err_alloc:
7532        free_event(event);
7533err_cpus:
7534        put_online_cpus();
7535err_task:
7536        if (task)
7537                put_task_struct(task);
7538err_group_fd:
7539        fdput(group);
7540err_fd:
7541        put_unused_fd(event_fd);
7542        return err;
7543}
7544
7545/**
7546 * perf_event_create_kernel_counter
7547 *
7548 * @attr: attributes of the counter to create
7549 * @cpu: cpu in which the counter is bound
7550 * @task: task to profile (NULL for percpu)
7551 */
7552struct perf_event *
7553perf_event_create_kernel_counter(struct perf_event_attr *attr, int cpu,
7554                                 struct task_struct *task,
7555                                 perf_overflow_handler_t overflow_handler,
7556                                 void *context)
7557{
7558        struct perf_event_context *ctx;
7559        struct perf_event *event;
7560        int err;
7561
7562        /*
7563         * Get the target context (task or percpu):
7564         */
7565
7566        event = perf_event_alloc(attr, cpu, task, NULL, NULL,
7567                                 overflow_handler, context);
7568        if (IS_ERR(event)) {
7569                err = PTR_ERR(event);
7570                goto err;
7571        }
7572
7573        /* Mark owner so we could distinguish it from user events. */
7574        event->owner = EVENT_OWNER_KERNEL;
7575
7576        account_event(event);
7577
7578        ctx = find_get_context(event->pmu, task, cpu);
7579        if (IS_ERR(ctx)) {
7580                err = PTR_ERR(ctx);
7581                goto err_free;
7582        }
7583
7584        WARN_ON_ONCE(ctx->parent_ctx);
7585        mutex_lock(&ctx->mutex);
7586        perf_install_in_context(ctx, event, cpu);
7587        perf_unpin_context(ctx);
7588        mutex_unlock(&ctx->mutex);
7589
7590        return event;
7591
7592err_free:
7593        free_event(event);
7594err:
7595        return ERR_PTR(err);
7596}
7597EXPORT_SYMBOL_GPL(perf_event_create_kernel_counter);
7598
7599void perf_pmu_migrate_context(struct pmu *pmu, int src_cpu, int dst_cpu)
7600{
7601        struct perf_event_context *src_ctx;
7602        struct perf_event_context *dst_ctx;
7603        struct perf_event *event, *tmp;
7604        LIST_HEAD(events);
7605
7606        src_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, src_cpu)->ctx;
7607        dst_ctx = &per_cpu_ptr(pmu->pmu_cpu_context, dst_cpu)->ctx;
7608
7609        mutex_lock(&src_ctx->mutex);
7610        list_for_each_entry_safe(event, tmp, &src_ctx->event_list,
7611                                 event_entry) {
7612                perf_remove_from_context(event, false);
7613                unaccount_event_cpu(event, src_cpu);
7614                put_ctx(src_ctx);
7615                list_add(&event->migrate_entry, &events);
7616        }
7617        mutex_unlock(&src_ctx->mutex);
7618
7619        synchronize_rcu();
7620
7621        mutex_lock(&dst_ctx->mutex);
7622        list_for_each_entry_safe(event, tmp, &events, migrate_entry) {
7623                list_del(&event->migrate_entry);
7624                if (event->state >= PERF_EVENT_STATE_OFF)
7625                        event->state = PERF_EVENT_STATE_INACTIVE;
7626                account_event_cpu(event, dst_cpu);
7627                perf_install_in_context(dst_ctx, event, dst_cpu);
7628                get_ctx(dst_ctx);
7629        }
7630        mutex_unlock(&dst_ctx->mutex);
7631}
7632EXPORT_SYMBOL_GPL(perf_pmu_migrate_context);
7633
7634static void sync_child_event(struct perf_event *child_event,
7635                               struct task_struct *child)
7636{
7637        struct perf_event *parent_event = child_event->parent;
7638        u64 child_val;
7639
7640        if (child_event->attr.inherit_stat)
7641                perf_event_read_event(child_event, child);
7642
7643        child_val = perf_event_count(child_event);
7644
7645        /*
7646         * Add back the child's count to the parent's count:
7647         */
7648        atomic64_add(child_val, &parent_event->child_count);
7649        atomic64_add(child_event->total_time_enabled,
7650                     &parent_event->child_total_time_enabled);
7651        atomic64_add(child_event->total_time_running,
7652                     &parent_event->child_total_time_running);
7653
7654        /*
7655         * Remove this event from the parent's list
7656         */
7657        WARN_ON_ONCE(parent_event->ctx->parent_ctx);
7658        mutex_lock(&parent_event->child_mutex);
7659        list_del_init(&child_event->child_list);
7660        mutex_unlock(&parent_event->child_mutex);
7661
7662        /*
7663         * Make sure user/parent get notified, that we just
7664         * lost one event.
7665         */
7666        perf_event_wakeup(parent_event);
7667
7668        /*
7669         * Release the parent event, if this was the last
7670         * reference to it.
7671         */
7672        put_event(parent_event);
7673}
7674
7675static void
7676__perf_event_exit_task(struct perf_event *child_event,
7677                         struct perf_event_context *child_ctx,
7678                         struct task_struct *child)
7679{
7680        /*
7681         * Do not destroy the 'original' grouping; because of the context
7682         * switch optimization the original events could've ended up in a
7683         * random child task.
7684         *
7685         * If we were to destroy the original group, all group related
7686         * operations would cease to function properly after this random
7687         * child dies.
7688         *
7689         * Do destroy all inherited groups, we don't care about those
7690         * and being thorough is better.
7691         */
7692        perf_remove_from_context(child_event, !!child_event->parent);
7693
7694        /*
7695         * It can happen that the parent exits first, and has events
7696         * that are still around due to the child reference. These
7697         * events need to be zapped.
7698         */
7699        if (child_event->parent) {
7700                sync_child_event(child_event, child);
7701                free_event(child_event);
7702        } else {
7703                child_event->state = PERF_EVENT_STATE_EXIT;
7704                perf_event_wakeup(child_event);
7705        }
7706}
7707
7708static void perf_event_exit_task_context(struct task_struct *child, int ctxn)
7709{
7710        struct perf_event *child_event, *next;
7711        struct perf_event_context *child_ctx, *clone_ctx = NULL;
7712        unsigned long flags;
7713
7714        if (likely(!child->perf_event_ctxp[ctxn])) {
7715                perf_event_task(child, NULL, 0);
7716                return;
7717        }
7718
7719        local_irq_save(flags);
7720        /*
7721         * We can't reschedule here because interrupts are disabled,
7722         * and either child is current or it is a task that can't be
7723         * scheduled, so we are now safe from rescheduling changing
7724         * our context.
7725         */
7726        child_ctx = rcu_dereference_raw(child->perf_event_ctxp[ctxn]);
7727
7728        /*
7729         * Take the context lock here so that if find_get_context is
7730         * reading child->perf_event_ctxp, we wait until it has
7731         * incremented the context's refcount before we do put_ctx below.
7732         */
7733        raw_spin_lock(&child_ctx->lock);
7734        task_ctx_sched_out(child_ctx);
7735        child->perf_event_ctxp[ctxn] = NULL;
7736
7737        /*
7738         * If this context is a clone; unclone it so it can't get
7739         * swapped to another process while we're removing all
7740         * the events from it.
7741         */
7742        clone_ctx = unclone_ctx(child_ctx);
7743        update_context_time(child_ctx);
7744        raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
7745
7746        if (clone_ctx)
7747                put_ctx(clone_ctx);
7748
7749        /*
7750         * Report the task dead after unscheduling the events so that we
7751         * won't get any samples after PERF_RECORD_EXIT. We can however still
7752         * get a few PERF_RECORD_READ events.
7753         */
7754        perf_event_task(child, child_ctx, 0);
7755
7756        /*
7757         * We can recurse on the same lock type through:
7758         *
7759         *   __perf_event_exit_task()
7760         *     sync_child_event()
7761         *       put_event()
7762         *         mutex_lock(&ctx->mutex)
7763         *
7764         * But since its the parent context it won't be the same instance.
7765         */
7766        mutex_lock(&child_ctx->mutex);
7767
7768        list_for_each_entry_safe(child_event, next, &child_ctx->event_list, event_entry)
7769                __perf_event_exit_task(child_event, child_ctx, child);
7770
7771        mutex_unlock(&child_ctx->mutex);
7772
7773        put_ctx(child_ctx);
7774}
7775
7776/*
7777 * When a child task exits, feed back event values to parent events.
7778 */
7779void perf_event_exit_task(struct task_struct *child)
7780{
7781        struct perf_event *event, *tmp;
7782        int ctxn;
7783
7784        mutex_lock(&child->perf_event_mutex);
7785        list_for_each_entry_safe(event, tmp, &child->perf_event_list,
7786                                 owner_entry) {
7787                list_del_init(&event->owner_entry);
7788
7789                /*
7790                 * Ensure the list deletion is visible before we clear
7791                 * the owner, closes a race against perf_release() where
7792                 * we need to serialize on the owner->perf_event_mutex.
7793                 */
7794                smp_wmb();
7795                event->owner = NULL;
7796        }
7797        mutex_unlock(&child->perf_event_mutex);
7798
7799        for_each_task_context_nr(ctxn)
7800                perf_event_exit_task_context(child, ctxn);
7801}
7802
7803static void perf_free_event(struct perf_event *event,
7804                            struct perf_event_context *ctx)
7805{
7806        struct perf_event *parent = event->parent;
7807
7808        if (WARN_ON_ONCE(!parent))
7809                return;
7810
7811        mutex_lock(&parent->child_mutex);
7812        list_del_init(&event->child_list);
7813        mutex_unlock(&parent->child_mutex);
7814
7815        put_event(parent);
7816
7817        perf_group_detach(event);
7818        list_del_event(event, ctx);
7819        free_event(event);
7820}
7821
7822/*
7823 * free an unexposed, unused context as created by inheritance by
7824 * perf_event_init_task below, used by fork() in case of fail.
7825 */
7826void perf_event_free_task(struct task_struct *task)
7827{
7828        struct perf_event_context *ctx;
7829        struct perf_event *event, *tmp;
7830        int ctxn;
7831
7832        for_each_task_context_nr(ctxn) {
7833                ctx = task->perf_event_ctxp[ctxn];
7834                if (!ctx)
7835                        continue;
7836
7837                mutex_lock(&ctx->mutex);
7838again:
7839                list_for_each_entry_safe(event, tmp, &ctx->pinned_groups,
7840                                group_entry)
7841                        perf_free_event(event, ctx);
7842
7843                list_for_each_entry_safe(event, tmp, &ctx->flexible_groups,
7844                                group_entry)
7845                        perf_free_event(event, ctx);
7846
7847                if (!list_empty(&ctx->pinned_groups) ||
7848                                !list_empty(&ctx->flexible_groups))
7849                        goto again;
7850
7851                mutex_unlock(&ctx->mutex);
7852
7853                put_ctx(ctx);
7854        }
7855}
7856
7857void perf_event_delayed_put(struct task_struct *task)
7858{
7859        int ctxn;
7860
7861        for_each_task_context_nr(ctxn)
7862                WARN_ON_ONCE(task->perf_event_ctxp[ctxn]);
7863}
7864
7865/*
7866 * inherit a event from parent task to child task:
7867 */
7868static struct perf_event *
7869inherit_event(struct perf_event *parent_event,
7870              struct task_struct *parent,
7871              struct perf_event_context *parent_ctx,
7872              struct task_struct *child,
7873              struct perf_event *group_leader,
7874              struct perf_event_context *child_ctx)
7875{
7876        enum perf_event_active_state parent_state = parent_event->state;
7877        struct perf_event *child_event;
7878        unsigned long flags;
7879
7880        /*
7881         * Instead of creating recursive hierarchies of events,
7882         * we link inherited events back to the original parent,
7883         * which has a filp for sure, which we use as the reference
7884         * count:
7885         */
7886        if (parent_event->parent)
7887                parent_event = parent_event->parent;
7888
7889        child_event = perf_event_alloc(&parent_event->attr,
7890                                           parent_event->cpu,
7891                                           child,
7892                                           group_leader, parent_event,
7893                                           NULL, NULL);
7894        if (IS_ERR(child_event))
7895                return child_event;
7896
7897        if (is_orphaned_event(parent_event) ||
7898            !atomic_long_inc_not_zero(&parent_event->refcount)) {
7899                free_event(child_event);
7900                return NULL;
7901        }
7902
7903        get_ctx(child_ctx);
7904
7905        /*
7906         * Make the child state follow the state of the parent event,
7907         * not its attr.disabled bit.  We hold the parent's mutex,
7908         * so we won't race with perf_event_{en, dis}able_family.
7909         */
7910        if (parent_state >= PERF_EVENT_STATE_INACTIVE)
7911                child_event->state = PERF_EVENT_STATE_INACTIVE;
7912        else
7913                child_event->state = PERF_EVENT_STATE_OFF;
7914
7915        if (parent_event->attr.freq) {
7916                u64 sample_period = parent_event->hw.sample_period;
7917                struct hw_perf_event *hwc = &child_event->hw;
7918
7919                hwc->sample_period = sample_period;
7920                hwc->last_period   = sample_period;
7921
7922                local64_set(&hwc->period_left, sample_period);
7923        }
7924
7925        child_event->ctx = child_ctx;
7926        child_event->overflow_handler = parent_event->overflow_handler;
7927        child_event->overflow_handler_context
7928                = parent_event->overflow_handler_context;
7929
7930        /*
7931         * Precalculate sample_data sizes
7932         */
7933        perf_event__header_size(child_event);
7934        perf_event__id_header_size(child_event);
7935
7936        /*
7937         * Link it up in the child's context:
7938         */
7939        raw_spin_lock_irqsave(&child_ctx->lock, flags);
7940        add_event_to_ctx(child_event, child_ctx);
7941        raw_spin_unlock_irqrestore(&child_ctx->lock, flags);
7942
7943        /*
7944         * Link this into the parent event's child list
7945         */
7946        WARN_ON_ONCE(parent_event->ctx->parent_ctx);
7947        mutex_lock(&parent_event->child_mutex);
7948        list_add_tail(&child_event->child_list, &parent_event->child_list);
7949        mutex_unlock(&parent_event->child_mutex);
7950
7951        return child_event;
7952}
7953
7954static int inherit_group(struct perf_event *parent_event,
7955              struct task_struct *parent,
7956              struct perf_event_context *parent_ctx,
7957              struct task_struct *child,
7958              struct perf_event_context *child_ctx)
7959{
7960        struct perf_event *leader;
7961        struct perf_event *sub;
7962        struct perf_event *child_ctr;
7963
7964        leader = inherit_event(parent_event, parent, parent_ctx,
7965                                 child, NULL, child_ctx);
7966        if (IS_ERR(leader))
7967                return PTR_ERR(leader);
7968        list_for_each_entry(sub, &parent_event->sibling_list, group_entry) {
7969                child_ctr = inherit_event(sub, parent, parent_ctx,
7970                                            child, leader, child_ctx);
7971                if (IS_ERR(child_ctr))
7972                        return PTR_ERR(child_ctr);
7973        }
7974        return 0;
7975}
7976
7977static int
7978inherit_task_group(struct perf_event *event, struct task_struct *parent,
7979                   struct perf_event_context *parent_ctx,
7980                   struct task_struct *child, int ctxn,
7981                   int *inherited_all)
7982{
7983        int ret;
7984        struct perf_event_context *child_ctx;
7985
7986        if (!event->attr.inherit) {
7987                *inherited_all = 0;
7988                return 0;
7989        }
7990
7991        child_ctx = child->perf_event_ctxp[ctxn];
7992        if (!child_ctx) {
7993                /*
7994                 * This is executed from the parent task context, so
7995                 * inherit events that have been marked for cloning.
7996                 * First allocate and initialize a context for the
7997                 * child.
7998                 */
7999
8000                child_ctx = alloc_perf_context(parent_ctx->pmu, child);
8001                if (!child_ctx)
8002                        return -ENOMEM;
8003
8004                child->perf_event_ctxp[ctxn] = child_ctx;
8005        }
8006
8007        ret = inherit_group(event, parent, parent_ctx,
8008                            child, child_ctx);
8009
8010        if (ret)
8011                *inherited_all = 0;
8012
8013        return ret;
8014}
8015
8016/*
8017 * Initialize the perf_event context in task_struct
8018 */
8019static int perf_event_init_context(struct task_struct *child, int ctxn)
8020{
8021        struct perf_event_context *child_ctx, *parent_ctx;
8022        struct perf_event_context *cloned_ctx;
8023        struct perf_event *event;
8024        struct task_struct *parent = current;
8025        int inherited_all = 1;
8026        unsigned long flags;
8027        int ret = 0;
8028
8029        if (likely(!parent->perf_event_ctxp[ctxn]))
8030                return 0;
8031
8032        /*
8033         * If the parent's context is a clone, pin it so it won't get
8034         * swapped under us.
8035         */
8036        parent_ctx = perf_pin_task_context(parent, ctxn);
8037        if (!parent_ctx)
8038                return 0;
8039
8040        /*
8041         * No need to check if parent_ctx != NULL here; since we saw
8042         * it non-NULL earlier, the only reason for it to become NULL
8043         * is if we exit, and since we're currently in the middle of
8044         * a fork we can't be exiting at the same time.
8045         */
8046
8047        /*
8048         * Lock the parent list. No need to lock the child - not PID
8049         * hashed yet and not running, so nobody can access it.
8050         */
8051        mutex_lock(&parent_ctx->mutex);
8052
8053        /*
8054         * We dont have to disable NMIs - we are only looking at
8055         * the list, not manipulating it:
8056         */
8057        list_for_each_entry(event, &parent_ctx->pinned_groups, group_entry) {
8058                ret = inherit_task_group(event, parent, parent_ctx,
8059                                         child, ctxn, &inherited_all);
8060                if (ret)
8061                        goto out_unlock;
8062        }
8063
8064        /*
8065         * We can't hold ctx->lock when iterating the ->flexible_group list due
8066         * to allocations, but we need to prevent rotation because
8067         * rotate_ctx() will change the list from interrupt context.
8068         */
8069        raw_spin_lock_irqsave(&parent_ctx->lock, flags);
8070        parent_ctx->rotate_disable = 1;
8071        raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
8072
8073        list_for_each_entry(event, &parent_ctx->flexible_groups, group_entry) {
8074                ret = inherit_task_group(event, parent, parent_ctx,
8075                                         child, ctxn, &inherited_all);
8076                if (ret)
8077                        goto out_unlock;
8078        }
8079
8080        raw_spin_lock_irqsave(&parent_ctx->lock, flags);
8081        parent_ctx->rotate_disable = 0;
8082
8083        child_ctx = child->perf_event_ctxp[ctxn];
8084
8085        if (child_ctx && inherited_all) {
8086                /*
8087                 * Mark the child context as a clone of the parent
8088                 * context, or of whatever the parent is a clone of.
8089                 *
8090                 * Note that if the parent is a clone, the holding of
8091                 * parent_ctx->lock avoids it from being uncloned.
8092                 */
8093                cloned_ctx = parent_ctx->parent_ctx;
8094                if (cloned_ctx) {
8095                        child_ctx->parent_ctx = cloned_ctx;
8096                        child_ctx->parent_gen = parent_ctx->parent_gen;
8097                } else {
8098                        child_ctx->parent_ctx = parent_ctx;
8099                        child_ctx->parent_gen = parent_ctx->generation;
8100                }
8101                get_ctx(child_ctx->parent_ctx);
8102        }
8103
8104        raw_spin_unlock_irqrestore(&parent_ctx->lock, flags);
8105out_unlock:
8106        mutex_unlock(&parent_ctx->mutex);
8107
8108        perf_unpin_context(parent_ctx);
8109        put_ctx(parent_ctx);
8110
8111        return ret;
8112}
8113
8114/*
8115 * Initialize the perf_event context in task_struct
8116 */
8117int perf_event_init_task(struct task_struct *child)
8118{
8119        int ctxn, ret;
8120
8121        memset(child->perf_event_ctxp, 0, sizeof(child->perf_event_ctxp));
8122        mutex_init(&child->perf_event_mutex);
8123        INIT_LIST_HEAD(&child->perf_event_list);
8124
8125        for_each_task_context_nr(ctxn) {
8126                ret = perf_event_init_context(child, ctxn);
8127                if (ret) {
8128                        perf_event_free_task(child);
8129                        return ret;
8130                }
8131        }
8132
8133        return 0;
8134}
8135
8136static void __init perf_event_init_all_cpus(void)
8137{
8138        struct swevent_htable *swhash;
8139        int cpu;
8140
8141        for_each_possible_cpu(cpu) {
8142                swhash = &per_cpu(swevent_htable, cpu);
8143                mutex_init(&swhash->hlist_mutex);
8144                INIT_LIST_HEAD(&per_cpu(rotation_list, cpu));
8145        }
8146}
8147
8148static void perf_event_init_cpu(int cpu)
8149{
8150        struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
8151
8152        mutex_lock(&swhash->hlist_mutex);
8153        swhash->online = true;
8154        if (swhash->hlist_refcount > 0) {
8155                struct swevent_hlist *hlist;
8156
8157                hlist = kzalloc_node(sizeof(*hlist), GFP_KERNEL, cpu_to_node(cpu));
8158                WARN_ON(!hlist);
8159                rcu_assign_pointer(swhash->swevent_hlist, hlist);
8160        }
8161        mutex_unlock(&swhash->hlist_mutex);
8162}
8163
8164#if defined CONFIG_HOTPLUG_CPU || defined CONFIG_KEXEC
8165static void perf_pmu_rotate_stop(struct pmu *pmu)
8166{
8167        struct perf_cpu_context *cpuctx = this_cpu_ptr(pmu->pmu_cpu_context);
8168
8169        WARN_ON(!irqs_disabled());
8170
8171        list_del_init(&cpuctx->rotation_list);
8172}
8173
8174static void __perf_event_exit_context(void *__info)
8175{
8176        struct remove_event re = { .detach_group = true };
8177        struct perf_event_context *ctx = __info;
8178
8179        perf_pmu_rotate_stop(ctx->pmu);
8180
8181        rcu_read_lock();
8182        list_for_each_entry_rcu(re.event, &ctx->event_list, event_entry)
8183                __perf_remove_from_context(&re);
8184        rcu_read_unlock();
8185}
8186
8187static void perf_event_exit_cpu_context(int cpu)
8188{
8189        struct perf_event_context *ctx;
8190        struct pmu *pmu;
8191        int idx;
8192
8193        idx = srcu_read_lock(&pmus_srcu);
8194        list_for_each_entry_rcu(pmu, &pmus, entry) {
8195                ctx = &per_cpu_ptr(pmu->pmu_cpu_context, cpu)->ctx;
8196
8197                mutex_lock(&ctx->mutex);
8198                smp_call_function_single(cpu, __perf_event_exit_context, ctx, 1);
8199                mutex_unlock(&ctx->mutex);
8200        }
8201        srcu_read_unlock(&pmus_srcu, idx);
8202}
8203
8204static void perf_event_exit_cpu(int cpu)
8205{
8206        struct swevent_htable *swhash = &per_cpu(swevent_htable, cpu);
8207
8208        perf_event_exit_cpu_context(cpu);
8209
8210        mutex_lock(&swhash->hlist_mutex);
8211        swhash->online = false;
8212        swevent_hlist_release(swhash);
8213        mutex_unlock(&swhash->hlist_mutex);
8214}
8215#else
8216static inline void perf_event_exit_cpu(int cpu) { }
8217#endif
8218
8219static int
8220perf_reboot(struct notifier_block *notifier, unsigned long val, void *v)
8221{
8222        int cpu;
8223
8224        for_each_online_cpu(cpu)
8225                perf_event_exit_cpu(cpu);
8226
8227        return NOTIFY_OK;
8228}
8229
8230/*
8231 * Run the perf reboot notifier at the very last possible moment so that
8232 * the generic watchdog code runs as long as possible.
8233 */
8234static struct notifier_block perf_reboot_notifier = {
8235        .notifier_call = perf_reboot,
8236        .priority = INT_MIN,
8237};
8238
8239static int
8240perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
8241{
8242        unsigned int cpu = (long)hcpu;
8243
8244        switch (action & ~CPU_TASKS_FROZEN) {
8245
8246        case CPU_UP_PREPARE:
8247        case CPU_DOWN_FAILED:
8248                perf_event_init_cpu(cpu);
8249                break;
8250
8251        case CPU_UP_CANCELED:
8252        case CPU_DOWN_PREPARE:
8253                perf_event_exit_cpu(cpu);
8254                break;
8255        default:
8256                break;
8257        }
8258
8259        return NOTIFY_OK;
8260}
8261
8262void __init perf_event_init(void)
8263{
8264        int ret;
8265
8266        idr_init(&pmu_idr);
8267
8268        perf_event_init_all_cpus();
8269        init_srcu_struct(&pmus_srcu);
8270        perf_pmu_register(&perf_swevent, "software", PERF_TYPE_SOFTWARE);
8271        perf_pmu_register(&perf_cpu_clock, NULL, -1);
8272        perf_pmu_register(&perf_task_clock, NULL, -1);
8273        perf_tp_register();
8274        perf_cpu_notifier(perf_cpu_notify);
8275        register_reboot_notifier(&perf_reboot_notifier);
8276
8277        ret = init_hw_breakpoint();
8278        WARN(ret, "hw_breakpoint initialization failed with: %d", ret);
8279
8280        /* do not patch jump label more than once per second */
8281        jump_label_rate_limit(&perf_sched_events, HZ);
8282
8283        /*
8284         * Build time assertion that we keep the data_head at the intended
8285         * location.  IOW, validation we got the __reserved[] size right.
8286         */
8287        BUILD_BUG_ON((offsetof(struct perf_event_mmap_page, data_head))
8288                     != 1024);
8289}
8290
8291static int __init perf_event_sysfs_init(void)
8292{
8293        struct pmu *pmu;
8294        int ret;
8295
8296        mutex_lock(&pmus_lock);
8297
8298        ret = bus_register(&pmu_bus);
8299        if (ret)
8300                goto unlock;
8301
8302        list_for_each_entry(pmu, &pmus, entry) {
8303                if (!pmu->name || pmu->type < 0)
8304                        continue;
8305
8306                ret = pmu_dev_alloc(pmu);
8307                WARN(ret, "Failed to register pmu: %s, reason %d\n", pmu->name, ret);
8308        }
8309        pmu_bus_running = 1;
8310        ret = 0;
8311
8312unlock:
8313        mutex_unlock(&pmus_lock);
8314
8315        return ret;
8316}
8317device_initcall(perf_event_sysfs_init);
8318
8319#ifdef CONFIG_CGROUP_PERF
8320static struct cgroup_subsys_state *
8321perf_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
8322{
8323        struct perf_cgroup *jc;
8324
8325        jc = kzalloc(sizeof(*jc), GFP_KERNEL);
8326        if (!jc)
8327                return ERR_PTR(-ENOMEM);
8328
8329        jc->info = alloc_percpu(struct perf_cgroup_info);
8330        if (!jc->info) {
8331                kfree(jc);
8332                return ERR_PTR(-ENOMEM);
8333        }
8334
8335        return &jc->css;
8336}
8337
8338static void perf_cgroup_css_free(struct cgroup_subsys_state *css)
8339{
8340        struct perf_cgroup *jc = container_of(css, struct perf_cgroup, css);
8341
8342        free_percpu(jc->info);
8343        kfree(jc);
8344}
8345
8346static int __perf_cgroup_move(void *info)
8347{
8348        struct task_struct *task = info;
8349        perf_cgroup_switch(task, PERF_CGROUP_SWOUT | PERF_CGROUP_SWIN);
8350        return 0;
8351}
8352
8353static void perf_cgroup_attach(struct cgroup_subsys_state *css,
8354                               struct cgroup_taskset *tset)
8355{
8356        struct task_struct *task;
8357
8358        cgroup_taskset_for_each(task, tset)
8359                task_function_call(task, __perf_cgroup_move, task);
8360}
8361
8362static void perf_cgroup_exit(struct cgroup_subsys_state *css,
8363                             struct cgroup_subsys_state *old_css,
8364                             struct task_struct *task)
8365{
8366        /*
8367         * cgroup_exit() is called in the copy_process() failure path.
8368         * Ignore this case since the task hasn't ran yet, this avoids
8369         * trying to poke a half freed task state from generic code.
8370         */
8371        if (!(task->flags & PF_EXITING))
8372                return;
8373
8374        task_function_call(task, __perf_cgroup_move, task);
8375}
8376
8377struct cgroup_subsys perf_event_cgrp_subsys = {
8378        .css_alloc      = perf_cgroup_css_alloc,
8379        .css_free       = perf_cgroup_css_free,
8380        .exit           = perf_cgroup_exit,
8381        .attach         = perf_cgroup_attach,
8382};
8383#endif /* CONFIG_CGROUP_PERF */
Note: See TracBrowser for help on using the repository browser.