[PATCH v10 20/33] perf/core: Prioritize kernel-owned pinned events when scheduling groups
From: Colton Lewis
Date: Thu Nov 19 2026 - 13:54:04 EST
When scheduling event groups in visit_groups_merge(), pinned and
flexible groups are ordered by group_index (creation order). Normally,
kernel-owned pinned events such as the hardlockup NMI watchdog are
created at boot with group_index 0 and are scheduled ahead of userspace
events.
However, if the NMI watchdog is disabled and re-enabled at runtime
after a userspace pinned per-CPU event is already active, the
watchdog's group_index is larger than the userspace event's
group_index. When hardware counters are constrained (for example, when
PMU partitioning reserves counters for a guest and leaves only one
general-purpose counter for the host), the userspace pinned event
claims the remaining counter first and merge_sched_in() transitions the
kernel watchdog event into PERF_EVENT_STATE_ERROR.
Order pinned kernel-owned events (is_pinned_kernel_event()) ahead of
userspace pinned events in perf_event_groups_cmp() and
perf_less_group_idx() while leaving flexible groups ordered by
group_index for normal round-robin rotation.
Suggested-by: James Clark <james.clark@xxxxxxxxxx>
Signed-off-by: Colton Lewis <coltonlewis@xxxxxxxxxx>
---
kernel/events/core.c | 28 +++++++++++++++++++++-------
1 file changed, 21 insertions(+), 7 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index dbc42ded5f698..6b82029b8ebc5 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -237,11 +237,16 @@ class_perf_ctx_lock_constructor(struct perf_cpu_context *cpuctx,
#define TASK_TOMBSTONE ((void *)-1L)
-static bool is_kernel_event(struct perf_event *event)
+static bool is_kernel_event(const struct perf_event *event)
{
return READ_ONCE(event->owner) == TASK_TOMBSTONE;
}
+static bool is_pinned_kernel_event(const struct perf_event *event)
+{
+ return event->attr.pinned && is_kernel_event(event);
+}
+
static DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
struct perf_event_context *perf_cpu_task_ctx(void)
@@ -1781,7 +1786,8 @@ static inline struct cgroup *event_cgroup(const struct perf_event *event)
*/
static __always_inline int
perf_event_groups_cmp(const int left_cpu, const struct pmu *left_pmu,
- const struct cgroup *left_cgroup, const u64 left_group_index,
+ const struct cgroup *left_cgroup, const bool left_kernel,
+ const u64 left_group_index,
const struct perf_event *right)
{
if (left_cpu < right->cpu)
@@ -1824,6 +1830,9 @@ perf_event_groups_cmp(const int left_cpu, const struct pmu *left_pmu,
}
#endif
+ if (left_kernel != is_pinned_kernel_event(right))
+ return left_kernel ? -1 : 1;
+
if (left_group_index < right->group_index)
return -1;
if (left_group_index > right->group_index)
@@ -1839,7 +1848,8 @@ static inline bool __group_less(struct rb_node *a, const struct rb_node *b)
{
struct perf_event *e = __node_2_pe(a);
return perf_event_groups_cmp(e->cpu, e->pmu_ctx->pmu, event_cgroup(e),
- e->group_index, __node_2_pe(b)) < 0;
+ is_pinned_kernel_event(e), e->group_index,
+ __node_2_pe(b)) < 0;
}
struct __group_key {
@@ -1853,8 +1863,9 @@ static inline int __group_cmp(const void *key, const struct rb_node *node)
const struct __group_key *a = key;
const struct perf_event *b = __node_2_pe(node);
- /* partial/subtree match: @cpu, @pmu, @cgroup; ignore: @group_index */
- return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup, b->group_index, b);
+ /* partial/subtree match: @cpu, @pmu, @cgroup; ignore: @kernel, @group_index */
+ return perf_event_groups_cmp(a->cpu, a->pmu, a->cgroup,
+ is_pinned_kernel_event(b), b->group_index, b);
}
static inline int
@@ -1863,9 +1874,9 @@ __group_cmp_ignore_cgroup(const void *key, const struct rb_node *node)
const struct __group_key *a = key;
const struct perf_event *b = __node_2_pe(node);
- /* partial/subtree match: @cpu, @pmu, ignore: @cgroup, @group_index */
+ /* partial/subtree match: @cpu, @pmu, ignore: @cgroup, @kernel, @group_index */
return perf_event_groups_cmp(a->cpu, a->pmu, event_cgroup(b),
- b->group_index, b);
+ is_pinned_kernel_event(b), b->group_index, b);
}
/*
@@ -4024,6 +4035,9 @@ static bool perf_less_group_idx(const void *l, const void *r, void __always_unus
const struct perf_event *le = *(const struct perf_event **)l;
const struct perf_event *re = *(const struct perf_event **)r;
+ if (is_pinned_kernel_event(le) != is_pinned_kernel_event(re))
+ return is_pinned_kernel_event(le);
+
return le->group_index < re->group_index;
}
--
2.56.1.708.g77570a6a1d-goog