[PATCH 2/3] perf/core: introduce pmu_event_flags and PMUEF_READ_CPU_PKG

[David Carrillo-Cisneros]

Introduce the flag PMUEF_READ_CPU_PKG, useful for uncore events, that
allows a PMU to signal the generic perf code that an event is readable
on the current CPU if the event is:
  - active in a CPU in the same package as the current CPU (local CPU)
  - not active but is attached to a CPU (i.e. event->cpu != -1) in the
  same package as the current CPU.

This is an optimization that avoids a unnecessary IPI for the common case
where uncore events are run and read in the same package but in
different CPUs.

As an example, the IPI removal speeds up perf_read in my Haswell system
as follows:
  - For event UNC_C_LLC_LOOKUP: From 260 us to 31 us.
  - For event RAPL's power/energy-cores/: From to 255 us to 27 us.

To store the flag PMUEF_READ_CPU_PKG, a new field, pmu_event_flags is
added to perf_event struct. The flags in this field are intended to be set
by the PMU in a per-event basis, allowing different events in the same PMU
to have different flags on. This patch does not demonstrate that events in
a PMU can have distinct flags, but the Intel CQM/CMT patch series does
makes use of this and uses the PMUEF_READ_CPU_PKG in a per-event basis

Signed-off-by: David Carrillo-Cisneros <davidcc@xxxxxxxxxx>
Reviewed-by: Stephane Eranian <eranian@xxxxxxxxxx>
---
 include/linux/perf_event.h | 12 ++++++++++
 kernel/events/core.c       | 55 +++++++++++++++++++++++++++++++++++++---------
 2 files changed, 57 insertions(+), 10 deletions(-)

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 7921f4f..d026166 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -681,9 +681,21 @@ struct perf_event {
 #endif
 
 	struct list_head		sb_list;
+
+	/* Per-event flags to generic code set by PMU. */
+	int				pmu_event_flags;
+
 #endif /* CONFIG_PERF_EVENTS */
 };
 
+/*
+ * Flags for pmu_event_flags.
+ *
+ * PMUEF_READ_CPU_PKG: A CPU event (or cgroup event) that can be read in
+ *     any CPU in event->cpu's package, even if inactive.
+ */
+#define PMUEF_READ_CPU_PKG		BIT(0)
+
 /**
  * struct perf_event_context - event context structure
  *
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 1c30f52..c2c6ad0 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -3336,6 +3336,39 @@ struct perf_read_data {
 	int ret;
 };
 
+static inline bool can_read_inactive(struct perf_event *event)
+{
+	return event->cpu >= 0 &&
+		(event->pmu_event_flags & PMUEF_READ_CPU_PKG);
+}
+
+static int find_cpu_to_read(struct perf_event *event)
+{
+	bool active = event->state == PERF_EVENT_STATE_ACTIVE;
+	int event_cpu = active ? event->oncpu : event->cpu;
+	int local_cpu = smp_processor_id();
+	u16 local_pkg, event_pkg;
+
+	if (event_cpu < 0)
+		return -1;
+
+	if (event->pmu_event_flags & PMUEF_READ_CPU_PKG) {
+		/*
+		 * Event with PMUEF_READ_CPU_PKG can be read in local CPU if:
+		 *   - CPU where event runs is on local CPU's pkg, or
+		 *   - event is bound to a CPU on local CPU's pkg
+		 *     (even if event is not in active state).
+		 */
+		event_pkg =  topology_physical_package_id(event_cpu);
+		local_pkg =  topology_physical_package_id(local_cpu);
+
+		if (event_pkg == local_pkg)
+			return local_cpu;
+	}
+
+	return event_cpu;
+}
+
 /*
  * Cross CPU call to read the hardware event
  */
@@ -3346,6 +3379,7 @@ static void __perf_event_read(void *info)
 	struct perf_event_context *ctx = event->ctx;
 	struct perf_cpu_context *cpuctx = __get_cpu_context(ctx);
 	struct pmu *pmu = event->pmu;
+	bool active, read_inactive = can_read_inactive(event);
 
 	/*
 	 * If this is a task context, we need to check whether it is
@@ -3354,7 +3388,7 @@ static void __perf_event_read(void *info)
 	 * event->count would have been updated to a recent sample
 	 * when the event was scheduled out.
 	 */
-	if (ctx->task && cpuctx->task_ctx != ctx)
+	if (ctx->task && cpuctx->task_ctx != ctx && !read_inactive)
 		return;
 
 	raw_spin_lock(&ctx->lock);
@@ -3364,7 +3398,9 @@ static void __perf_event_read(void *info)
 	}
 
 	update_event_times(event);
-	if (event->state != PERF_EVENT_STATE_ACTIVE)
+
+	active = event->state == PERF_EVENT_STATE_ACTIVE;
+	if (!active && !read_inactive)
 		goto unlock;
 
 	if (!data->group) {
@@ -3379,7 +3415,8 @@ static void __perf_event_read(void *info)
 
 	list_for_each_entry(sub, &event->sibling_list, group_entry) {
 		update_event_times(sub);
-		if (sub->state == PERF_EVENT_STATE_ACTIVE) {
+		active = sub->state == PERF_EVENT_STATE_ACTIVE;
+		if (active || can_read_inactive(sub)) {
 			/*
 			 * Use sibling's PMU rather than @event's since
 			 * sibling could be on different (eg: software) PMU.
@@ -3457,19 +3494,17 @@ u64 perf_event_read_local(struct perf_event *event)
 
 static int perf_event_read(struct perf_event *event, bool group)
 {
-	int ret = 0;
+	int ret = 0, cpu_to_read;
 
-	/*
-	 * If event is enabled and currently active on a CPU, update the
-	 * value in the event structure:
-	 */
-	if (event->state == PERF_EVENT_STATE_ACTIVE) {
+	cpu_to_read = find_cpu_to_read(event);
+
+	if (cpu_to_read >= 0) {
 		struct perf_read_data data = {
 			.event = event,
 			.group = group,
 			.ret = 0,
 		};
-		ret = smp_call_function_single(event->oncpu,
+		ret = smp_call_function_single(cpu_to_read,
 					       __perf_event_read, &data, 1);
 		ret = ret ? : data.ret;
 	} else if (event->state == PERF_EVENT_STATE_INACTIVE) {
-- 
2.8.0.rc3.226.g39d4020