[Patch v3 8/8] perf/x86/intel: Prevent drain_pebs() reentry

[Dapeng Mi]

The PEBS buffer is shared by all events on a CPU, so drain_pebs() must
not run concurrently. If it is reentered, one instance may observe stale
buffer state and potentially access out-of-bound memory.

Most invocations happen in NMI context, which naturally prevents reentry.
However, drain_pebs() is also reachable from process context via
intel_pmu_drain_pebs_buffer().

In those paths, the PMU is often already disabled, but not guaranteed.
For example, __intel_pmu_pebs_disable() only disables the target counter,
so other active counters can still raise a PMI and interrupt an in-flight
drain_pebs().

Introduce __intel_pmu_quiesce() and __intel_pmu_resume() helpers and
use them in intel_pmu_drain_pebs_buffer() to disable the full PMU
around the drain_pebs() call, preventing reentry.

Signed-off-by: Dapeng Mi <dapeng1.mi@xxxxxxxxxxxxxxx>
---
 arch/x86/events/intel/core.c | 40 ++++++++++++++++++++++++++++--------
 arch/x86/events/intel/ds.c   |  7 -------
 2 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/arch/x86/events/intel/core.c b/arch/x86/events/intel/core.c
index 361f8e0ab36c..726b39b9bba9 100644
--- a/arch/x86/events/intel/core.c
+++ b/arch/x86/events/intel/core.c
@@ -3125,6 +3125,36 @@ static void intel_pmu_del_event(struct perf_event *event)
 		this_cpu_ptr(&cpu_hw_events)->n_late_setup--;
 }
 
+static inline void __intel_pmu_quiesce(bool pmu_enabled)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+	cpuc->enabled = 0;
+	if (pmu_enabled)
+		intel_pmu_disable_all();
+}
+
+static inline void __intel_pmu_resume(bool pmu_enabled)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+
+	cpuc->enabled = pmu_enabled;
+	if (pmu_enabled)
+		intel_pmu_enable_all(0);
+}
+
+void intel_pmu_drain_pebs_buffer(void)
+{
+	struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
+	bool pmu_enabled = cpuc->enabled;
+	struct perf_sample_data data;
+
+	/* Disable PMU so no new PMI can interrupt and re-enter drain_pebs(). */
+	__intel_pmu_quiesce(pmu_enabled);
+	static_call(x86_pmu_drain_pebs)(NULL, &data);
+	__intel_pmu_resume(pmu_enabled);
+}
+
 static int icl_set_topdown_event_period(struct perf_event *event)
 {
 	struct hw_perf_event *hwc = &event->hw;
@@ -3322,10 +3352,7 @@ static void intel_pmu_read_event(struct perf_event *event)
 		if (is_metric_event(event) && (cpuc->txn_flags & PERF_PMU_TXN_READ))
 			return;
 
-		cpuc->enabled = 0;
-		if (pmu_enabled)
-			intel_pmu_disable_all();
-
+		__intel_pmu_quiesce(pmu_enabled);
 		/*
 		 * If the PEBS counters snapshotting is enabled,
 		 * the topdown event is available in PEBS records.
@@ -3334,10 +3361,7 @@ static void intel_pmu_read_event(struct perf_event *event)
 			static_call(intel_pmu_update_topdown_event)(event, NULL);
 		else
 			intel_pmu_drain_pebs_buffer();
-
-		cpuc->enabled = pmu_enabled;
-		if (pmu_enabled)
-			intel_pmu_enable_all(0);
+		__intel_pmu_resume(pmu_enabled);
 
 		return;
 	}
diff --git a/arch/x86/events/intel/ds.c b/arch/x86/events/intel/ds.c
index e86e4ba91e1b..7f8b98d1837d 100644
--- a/arch/x86/events/intel/ds.c
+++ b/arch/x86/events/intel/ds.c
@@ -1240,13 +1240,6 @@ int intel_pmu_drain_bts_buffer(void)
 	return 1;
 }
 
-void intel_pmu_drain_pebs_buffer(void)
-{
-	struct perf_sample_data data;
-
-	static_call(x86_pmu_drain_pebs)(NULL, &data);
-}
-
 /*
  * PEBS
  */
-- 
2.34.1