[PATCH v6 3/3]: perf/core: add mux switch to skip to the current CPU's events list on mux interrupt

From: Alexey Budankov
Date: Wed Aug 02 2017 - 04:17:08 EST

This patch implements mux switch that triggers skipping to the current CPU's
events list at mulitplexing hrtimer interrupt handler as well as adoption of
the switch in the existing implementation.

perf_event_groups_iterate_cpu() API is introduced to implement iteration thru the
certain CPU groups list skipping groups allocated for the other CPUs.

Signed-off-by: Alexey Budankov <alexey.budankov@xxxxxxxxxxxxxxx>
---
kernel/events/core.c | 159 ++++++++++++++++++++++++++++++++++++++-------------
1 file changed, 118 insertions(+), 41 deletions(-)

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 5ccb8a2..61f370e 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -556,11 +556,11 @@ void perf_sample_event_took(u64 sample_len_ns)
static atomic64_t perf_event_id;

static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
- enum event_type_t event_type);
+ enum event_type_t event_type, int mux);

static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
enum event_type_t event_type,
- struct task_struct *task);
+ struct task_struct *task, int mux);

static void update_context_time(struct perf_event_context *ctx);
static u64 perf_event_time(struct perf_event *event);
@@ -702,6 +702,7 @@ static void perf_cgroup_switch(struct task_struct *task, int mode)
struct perf_cpu_context *cpuctx;
struct list_head *list;
unsigned long flags;
+ int mux = 0;

/*
* Disable interrupts and preemption to avoid this CPU's
@@ -717,7 +718,7 @@ static void perf_cgroup_switch(struct task_struct *task, int mode)
perf_pmu_disable(cpuctx->ctx.pmu);

if (mode & PERF_CGROUP_SWOUT) {
- cpu_ctx_sched_out(cpuctx, EVENT_ALL);
+ cpu_ctx_sched_out(cpuctx, EVENT_ALL, mux);
/*
* must not be done before ctxswout due
* to event_filter_match() in event_sched_out()
@@ -736,7 +737,7 @@ static void perf_cgroup_switch(struct task_struct *task, int mode)
*/
cpuctx->cgrp = perf_cgroup_from_task(task,
&cpuctx->ctx);
- cpu_ctx_sched_in(cpuctx, EVENT_ALL, task);
+ cpu_ctx_sched_in(cpuctx, EVENT_ALL, task, mux);
}
perf_pmu_enable(cpuctx->ctx.pmu);
perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
@@ -1611,6 +1612,36 @@ perf_event_groups_rotate(struct rb_root *groups, int cpu)
typedef int(*perf_event_groups_iterate_f)(struct perf_event *, void *);

/*
+ * Find group_list list by a cpu key and call provided callback for every
+ * group on the list.
+ */
+static void
+perf_event_groups_iterate_cpu(struct rb_root *groups, int cpu,
+ perf_event_groups_iterate_f callback, void *data)
+{
+ struct rb_node *node;
+ struct perf_event *event, *node_event;
+
+ node = groups->rb_node;
+
+ while (node) {
+ node_event = container_of(node,
+ struct perf_event, group_node);
+
+ if (cpu < node_event->cpu) {
+ node = node->rb_left;
+ } else if (cpu > node_event->cpu) {
+ node = node->rb_right;
+ } else {
+ list_for_each_entry(event, &node_event->group_list,
+ group_entry)
+ callback(event, data);
+ break;
+ }
+ }
+}
+
+/*
* Iterate event groups and call provided callback for every group in the tree.
* Iteration stops if the callback returns non zero.
*/
@@ -1866,7 +1897,6 @@ list_del_event(struct perf_event *event, struct perf_event_context *ctx)
static void perf_group_detach(struct perf_event *event)
{
struct perf_event *sibling, *tmp;
- struct list_head *list = NULL;

lockdep_assert_held(&event->ctx->lock);

@@ -2408,36 +2438,38 @@ static void add_event_to_ctx(struct perf_event *event,

static void ctx_sched_out(struct perf_event_context *ctx,
struct perf_cpu_context *cpuctx,
- enum event_type_t event_type);
+ enum event_type_t event_type, int mux);
static void
ctx_sched_in(struct perf_event_context *ctx,
struct perf_cpu_context *cpuctx,
enum event_type_t event_type,
- struct task_struct *task);
+ struct task_struct *task, int mux);

static void task_ctx_sched_out(struct perf_cpu_context *cpuctx,
struct perf_event_context *ctx,
enum event_type_t event_type)
{
+ int mux = 0;
+
if (!cpuctx->task_ctx)
return;

if (WARN_ON_ONCE(ctx != cpuctx->task_ctx))
return;

- ctx_sched_out(ctx, cpuctx, event_type);
+ ctx_sched_out(ctx, cpuctx, event_type, mux);
}

static void perf_event_sched_in(struct perf_cpu_context *cpuctx,
struct perf_event_context *ctx,
- struct task_struct *task)
+ struct task_struct *task, int mux)
{
- cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task);
+ cpu_ctx_sched_in(cpuctx, EVENT_PINNED, task, mux);
if (ctx)
- ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task);
- cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task);
+ ctx_sched_in(ctx, cpuctx, EVENT_PINNED, task, mux);
+ cpu_ctx_sched_in(cpuctx, EVENT_FLEXIBLE, task, mux);
if (ctx)
- ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task);
+ ctx_sched_in(ctx, cpuctx, EVENT_FLEXIBLE, task, mux);
}

/*
@@ -2461,6 +2493,7 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,
{
enum event_type_t ctx_event_type = event_type & EVENT_ALL;
bool cpu_event = !!(event_type & EVENT_CPU);
+ int mux = 0;

/*
* If pinned groups are involved, flexible groups also need to be
@@ -2481,11 +2514,11 @@ static void ctx_resched(struct perf_cpu_context *cpuctx,
* - otherwise, do nothing more.
*/
if (cpu_event)
- cpu_ctx_sched_out(cpuctx, ctx_event_type);
+ cpu_ctx_sched_out(cpuctx, ctx_event_type, mux);
else if (ctx_event_type & EVENT_PINNED)
- cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
+ cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE, mux);

- perf_event_sched_in(cpuctx, task_ctx, current);
+ perf_event_sched_in(cpuctx, task_ctx, current, mux);
perf_pmu_enable(cpuctx->ctx.pmu);
}

@@ -2503,6 +2536,7 @@ static int __perf_install_in_context(void *info)
struct perf_event_context *task_ctx = cpuctx->task_ctx;
bool reprogram = true;
int ret = 0;
+ int mux = 0;

raw_spin_lock(&cpuctx->ctx.lock);
if (ctx->task) {
@@ -2529,7 +2563,7 @@ static int __perf_install_in_context(void *info)
}

if (reprogram) {
- ctx_sched_out(ctx, cpuctx, EVENT_TIME);
+ ctx_sched_out(ctx, cpuctx, EVENT_TIME, mux);
add_event_to_ctx(event, ctx);
ctx_resched(cpuctx, task_ctx, get_event_type(event));
} else {
@@ -2665,13 +2699,14 @@ static void __perf_event_enable(struct perf_event *event,
{
struct perf_event *leader = event->group_leader;
struct perf_event_context *task_ctx;
+ int mux = 0;

if (event->state >= PERF_EVENT_STATE_INACTIVE ||
event->state <= PERF_EVENT_STATE_ERROR)
return;

if (ctx->is_active)
- ctx_sched_out(ctx, cpuctx, EVENT_TIME);
+ ctx_sched_out(ctx, cpuctx, EVENT_TIME, mux);

__perf_event_mark_enabled(event);

@@ -2681,7 +2716,7 @@ static void __perf_event_enable(struct perf_event *event,
if (!event_filter_match(event)) {
if (is_cgroup_event(event))
perf_cgroup_defer_enabled(event);
- ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
+ ctx_sched_in(ctx, cpuctx, EVENT_TIME, current, mux);
return;
}

@@ -2690,7 +2725,7 @@ static void __perf_event_enable(struct perf_event *event,
* then don't put it on unless the group is on.
*/
if (leader != event && leader->state != PERF_EVENT_STATE_ACTIVE) {
- ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
+ ctx_sched_in(ctx, cpuctx, EVENT_TIME, current, mux);
return;
}

@@ -2886,13 +2921,14 @@ EXPORT_SYMBOL_GPL(perf_event_refresh);

static void ctx_sched_out(struct perf_event_context *ctx,
struct perf_cpu_context *cpuctx,
- enum event_type_t event_type)
+ enum event_type_t event_type, int mux)
{
int is_active = ctx->is_active;
struct group_sched_params params = {
.cpuctx = cpuctx,
.ctx = ctx
};
+ int cpu = smp_processor_id();

lockdep_assert_held(&ctx->lock);

@@ -2939,13 +2975,27 @@ static void ctx_sched_out(struct perf_event_context *ctx,

perf_pmu_disable(ctx->pmu);
if (is_active & EVENT_PINNED) {
- perf_event_groups_iterate(&ctx->pinned_groups,
- group_sched_out_callback, &params);
+ if (mux) {
+ perf_event_groups_iterate_cpu(&ctx->pinned_groups, -1,
+ group_sched_out_callback, &params);
+ perf_event_groups_iterate_cpu(&ctx->pinned_groups, cpu,
+ group_sched_out_callback, &params);
+ } else {
+ perf_event_groups_iterate(&ctx->pinned_groups,
+ group_sched_out_callback, &params);
+ }
}

if (is_active & EVENT_FLEXIBLE) {
- perf_event_groups_iterate(&ctx->flexible_groups,
- group_sched_out_callback, &params);
+ if (mux) {
+ perf_event_groups_iterate_cpu(&ctx->flexible_groups, -1,
+ group_sched_out_callback, &params);
+ perf_event_groups_iterate_cpu(&ctx->flexible_groups, cpu,
+ group_sched_out_callback, &params);
+ } else {
+ perf_event_groups_iterate(&ctx->flexible_groups,
+ group_sched_out_callback, &params);
+ }
}
perf_pmu_enable(ctx->pmu);
}
@@ -3234,9 +3284,9 @@ void __perf_event_task_sched_out(struct task_struct *task,
* Called with IRQs disabled
*/
static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
- enum event_type_t event_type)
+ enum event_type_t event_type, int mux)
{
- ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
+ ctx_sched_out(&cpuctx->ctx, cpuctx, event_type, mux);
}

static int
@@ -3305,13 +3355,14 @@ static void
ctx_sched_in(struct perf_event_context *ctx,
struct perf_cpu_context *cpuctx,
enum event_type_t event_type,
- struct task_struct *task)
+ struct task_struct *task, int mux)
{
int is_active = ctx->is_active;
struct group_sched_params params = {
.cpuctx = cpuctx,
.ctx = ctx
};
+ int cpu = smp_processor_id();

lockdep_assert_held(&ctx->lock);

@@ -3339,31 +3390,55 @@ ctx_sched_in(struct perf_event_context *ctx,
* First go through the list and put on any pinned groups
* in order to give them the best chance of going on.
*/
- if (is_active & EVENT_PINNED)
- perf_event_groups_iterate(&ctx->pinned_groups,
- ctx_pinned_sched_in, &params);
+ if (is_active & EVENT_PINNED) {
+ if (mux) {
+ perf_event_groups_iterate_cpu(&ctx->pinned_groups,
+ -1, ctx_pinned_sched_in,
+ &params);
+ perf_event_groups_iterate_cpu(&ctx->pinned_groups,
+ cpu, ctx_pinned_sched_in,
+ &params);
+ } else {
+ perf_event_groups_iterate(&ctx->pinned_groups,
+ ctx_pinned_sched_in,
+ &params);
+ }
+ }

/* Then walk through the lower prio flexible groups */
if (is_active & EVENT_FLEXIBLE) {
- params.can_add_hw = 1;
- perf_event_groups_iterate(&ctx->flexible_groups,
- ctx_flexible_sched_in, &params);
+ if (mux) {
+ params.can_add_hw = 1;
+ perf_event_groups_iterate_cpu(&ctx->flexible_groups,
+ -1, ctx_flexible_sched_in,
+ &params);
+ params.can_add_hw = 1;
+ perf_event_groups_iterate_cpu(&ctx->flexible_groups,
+ cpu, ctx_flexible_sched_in,
+ &params);
+ } else {
+ params.can_add_hw = 1;
+ perf_event_groups_iterate(&ctx->flexible_groups,
+ ctx_flexible_sched_in,
+ &params);
+ }
}
}

static void cpu_ctx_sched_in(struct perf_cpu_context *cpuctx,
enum event_type_t event_type,
- struct task_struct *task)
+ struct task_struct *task, int mux)
{
struct perf_event_context *ctx = &cpuctx->ctx;

- ctx_sched_in(ctx, cpuctx, event_type, task);
+ ctx_sched_in(ctx, cpuctx, event_type, task, mux);
}

static void perf_event_context_sched_in(struct perf_event_context *ctx,
struct task_struct *task)
{
struct perf_cpu_context *cpuctx;
+ int mux = 0;

cpuctx = __get_cpu_context(ctx);
if (cpuctx->task_ctx == ctx)
@@ -3626,6 +3701,7 @@ static int perf_rotate_context(struct perf_cpu_context *cpuctx)
{
struct perf_event_context *ctx = NULL;
int rotate = 0;
+ int mux = 1;

if (cpuctx->ctx.nr_events) {
if (cpuctx->ctx.nr_events != cpuctx->ctx.nr_active)
@@ -3644,15 +3720,15 @@ static int perf_rotate_context(struct perf_cpu_context *cpuctx)
perf_ctx_lock(cpuctx, cpuctx->task_ctx);
perf_pmu_disable(cpuctx->ctx.pmu);

- cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE);
+ cpu_ctx_sched_out(cpuctx, EVENT_FLEXIBLE, mux);
if (ctx)
- ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE);
+ ctx_sched_out(ctx, cpuctx, EVENT_FLEXIBLE, mux);

rotate_ctx(&cpuctx->ctx);
if (ctx)
rotate_ctx(ctx);

- perf_event_sched_in(cpuctx, ctx, current);
+ perf_event_sched_in(cpuctx, ctx, current, mux);

perf_pmu_enable(cpuctx->ctx.pmu);
perf_ctx_unlock(cpuctx, cpuctx->task_ctx);
@@ -3704,6 +3780,7 @@ static void perf_event_enable_on_exec(int ctxn)
struct perf_event *event;
unsigned long flags;
int enabled = 0;
+ int mux = 0;

local_irq_save(flags);
ctx = current->perf_event_ctxp[ctxn];
@@ -3712,7 +3789,7 @@ static void perf_event_enable_on_exec(int ctxn)

cpuctx = __get_cpu_context(ctx);
perf_ctx_lock(cpuctx, ctx);
- ctx_sched_out(ctx, cpuctx, EVENT_TIME);
+ ctx_sched_out(ctx, cpuctx, EVENT_TIME, mux);
list_for_each_entry(event, &ctx->event_list, event_entry) {
enabled |= event_enable_on_exec(event, ctx);
event_type |= get_event_type(event);
@@ -3725,7 +3802,7 @@ static void perf_event_enable_on_exec(int ctxn)
clone_ctx = unclone_ctx(ctx);
ctx_resched(cpuctx, ctx, event_type);
} else {
- ctx_sched_in(ctx, cpuctx, EVENT_TIME, current);
+ ctx_sched_in(ctx, cpuctx, EVENT_TIME, current, mux);
}
perf_ctx_unlock(cpuctx, ctx);