[PATCH V2 4/4] perf cgroup: Add fast path for cgroup switch

[kan . liang]

From: Kan Liang <kan.liang@xxxxxxxxxxxxxxx>

Generic visit_groups_merge() is used in cgroup context switch to sched
in cgroup events, which has high overhead especially in frequent context
switch with several events and cgroups involved. Because it feeds all
events on a given CPU to pinned/flexible_sched_in() regardless the
cgroup.

Add a fast path to only feed the specific cgroup events to
pinned/flexible_sched_in() in cgroup context switch for non-multiplexing
case.

Don't need event_filter_match() to filter cgroup and CPU in fast path.
Only pmu_filter_match() is enough.

Don't need to specially handle system-wide event for fast path. Move it
to slow path.

Signed-off-by: Kan Liang <kan.liang@xxxxxxxxxxxxxxx>
---
 kernel/events/core.c | 92 ++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 75 insertions(+), 17 deletions(-)

diff --git a/kernel/events/core.c b/kernel/events/core.c
index 6891c74..67b0135 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -1780,6 +1780,20 @@ perf_event_groups_first_cgroup(struct perf_event_groups *groups,
 	return match;
 }
 
+static struct perf_event *
+perf_event_groups_next_cgroup(struct perf_event *event)
+{
+	struct perf_event *next;
+
+	next = rb_entry_safe(rb_next(&event->group_node), typeof(*event), group_node);
+	if (next && (next->cpu == event->cpu) &&
+	    (next->cgrp_group_index == event->cgrp_group_index) &&
+	    (next->cgrp_id == event->cgrp_id))
+		return next;
+
+	return NULL;
+}
+
 static void
 perf_event_groups_insert(struct perf_event_groups *groups,
 			 struct perf_event *event)
@@ -3464,13 +3478,69 @@ static void cpu_ctx_sched_out(struct perf_cpu_context *cpuctx,
 	ctx_sched_out(&cpuctx->ctx, cpuctx, event_type);
 }
 
+struct sched_in_data {
+	struct perf_event_context *ctx;
+	struct perf_cpu_context *cpuctx;
+	int can_add_hw;
+	enum event_type_t event_type;
+};
+
+#ifdef CONFIG_CGROUP_PERF
+
+static void cgroup_visit_groups(struct perf_event *evt, void *data,
+				int (*func)(struct perf_event *, void *, int (*)(struct perf_event *)))
+{
+	while (evt) {
+		if (func(evt, (void *)data, pmu_filter_match))
+			break;
+		evt = perf_event_groups_next_cgroup(evt);
+	}
+}
+
+static int cgroup_visit_groups_merge(int cpu, void *data,
+				     int (*func)(struct perf_event *, void *, int (*)(struct perf_event *)))
+{
+	struct sched_in_data *sid = data;
+	struct cgroup_subsys_state *css;
+	struct perf_cgroup *cgrp;
+	struct perf_event *evt, *rotated_evt = NULL;
+
+	for (css = &sid->cpuctx->cgrp->css; css; css = css->parent) {
+		/* root cgroup doesn't have events */
+		if (css->id == 1)
+			return 0;
+
+		cgrp = container_of(css, struct perf_cgroup, css);
+		/* Only visit groups when the cgroup has events */
+		if (cgrp->cgrp_event_type & sid->event_type) {
+			if (CGROUP_PINNED(sid->event_type))
+				evt = *per_cpu_ptr(cgrp->pinned_event, cpu);
+			else {
+				evt = *per_cpu_ptr(cgrp->flexible_event, cpu);
+				rotated_evt = *per_cpu_ptr(cgrp->rotated_event, cpu);
+			}
+			cgroup_visit_groups(evt, data, func);
+			cgroup_visit_groups(rotated_evt, data, func);
+		}
+	}
+
+	return 0;
+}
+#endif
+
 static int visit_groups_merge(struct perf_event_groups *groups, int cpu,
 			      int (*func)(struct perf_event *, void *, int (*)(struct perf_event *)),
 			      void *data)
 {
 	struct perf_event **evt, *evt1, *evt2;
+	struct sched_in_data *sid = data;
 	int ret;
 
+#ifdef CONFIG_CGROUP_PERF
+	/* fast path for cgroup switch, not support multiplexing */
+	if ((sid->event_type) && !sid->cpuctx->hrtimer_active)
+		return cgroup_visit_groups_merge(cpu, data, func);
+#endif
 	evt1 = perf_event_groups_first(groups, -1);
 	evt2 = perf_event_groups_first(groups, cpu);
 
@@ -3486,23 +3556,17 @@ static int visit_groups_merge(struct perf_event_groups *groups, int cpu,
 			evt = &evt2;
 		}
 
-		ret = func(*evt, data, event_filter_match);
-		if (ret)
-			return ret;
-
+		if (!perf_cgroup_skip_switch(sid->event_type, *evt, CGROUP_PINNED(sid->event_type))) {
+			ret = func(*evt, data, event_filter_match);
+			if (ret)
+				return ret;
+		}
 		*evt = perf_event_groups_next(*evt);
 	}
 
 	return 0;
 }
 
-struct sched_in_data {
-	struct perf_event_context *ctx;
-	struct perf_cpu_context *cpuctx;
-	int can_add_hw;
-	enum event_type_t event_type;
-};
-
 static int pinned_sched_in(struct perf_event *event, void *data,
 			   int (*filter_match)(struct perf_event *))
 {
@@ -3511,9 +3575,6 @@ static int pinned_sched_in(struct perf_event *event, void *data,
 	if (event->state <= PERF_EVENT_STATE_OFF)
 		return 0;
 
-	if (perf_cgroup_skip_switch(sid->event_type, event, true))
-		return 0;
-
 	if (!filter_match(event))
 		return 0;
 
@@ -3540,9 +3601,6 @@ static int flexible_sched_in(struct perf_event *event, void *data,
 	if (event->state <= PERF_EVENT_STATE_OFF)
 		return 0;
 
-	if (perf_cgroup_skip_switch(sid->event_type, event, false))
-		return 0;
-
 	if (!filter_match(event))
 		return 0;
 
-- 
2.7.4