[PATCH v3 06/19] perf/core: Support deferred unwinding for BUILD_ID_OFFSET sample types
From: Ian Rogers
Date: Mon Nov 16 2026 - 02:45:25 EST
When attr.defer_callchain is set on events using
PERF_SAMPLE_BUILD_ID_OFFSET or PERF_SAMPLE_CALLCHAIN_BUILD_ID_OFFSET,
request a deferred user-space unwind via unwind_deferred_request() and
emit a PERF_CONTEXT_USER_DEFERRED marker and cookie in the sample.
In the faultable task_work callback (perf_unwind_deferred_callback()),
resolve the unwound user IPs to build IDs and file offsets using
stack_map_get_build_id_offset(..., may_fault=true), encode them into the
compressed build ID + offset format, and emit
PERF_RECORD_CALLCHAIN_DEFERRED with PERF_RECORD_MISC_MMAP_BUILD_ID set.
Suggested-by: Namhyung Kim <namhyung@xxxxxxxxxx>
Signed-off-by: Ian Rogers <irogers@xxxxxxxxxx>
---
kernel/events/core.c | 175 ++++++++++++++++++++++++++++++++++++++++---
1 file changed, 164 insertions(+), 11 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index d38ffe52c301..ef40c4442e82 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -460,6 +460,8 @@ static atomic_t nr_bpf_events __read_mostly;
static atomic_t nr_cgroup_events __read_mostly;
static atomic_t nr_text_poke_events __read_mostly;
static atomic_t nr_build_id_events __read_mostly;
+static atomic_t nr_defer_output_build_id_events __read_mostly;
+static atomic_t nr_defer_output_callchain_build_id_events __read_mostly;
static LIST_HEAD(pmus);
static DEFINE_MUTEX(pmus_lock);
@@ -5639,6 +5641,25 @@ static void detach_perf_ctx_data(struct perf_event *event)
}
}
+static bool has_defer_output_callchain_build_id_offset(const struct perf_event *event)
+{
+ return event->attr.defer_output &&
+ (event->attr.sample_type & PERF_SAMPLE_CALLCHAIN_BUILD_ID_OFFSET) &&
+ !(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) &&
+ !event->attr.exclude_callchain_user;
+}
+
+static bool has_defer_output_build_id_offset(const struct perf_event *event)
+{
+ if (!event->attr.defer_output)
+ return false;
+ if (has_defer_output_callchain_build_id_offset(event))
+ return true;
+ return (event->attr.sample_type & PERF_SAMPLE_BUILD_ID_OFFSET) &&
+ (!(event->attr.sample_type & PERF_SAMPLE_CALLCHAIN) ||
+ event->attr.exclude_callchain_user);
+}
+
static void unaccount_event(struct perf_event *event)
{
bool dec = false;
@@ -5652,6 +5673,10 @@ static void unaccount_event(struct perf_event *event)
atomic_dec(&nr_mmap_events);
if (event->attr.build_id)
atomic_dec(&nr_build_id_events);
+ if (has_defer_output_build_id_offset(event))
+ atomic_dec(&nr_defer_output_build_id_events);
+ if (has_defer_output_callchain_build_id_offset(event))
+ atomic_dec(&nr_defer_output_callchain_build_id_events);
if (event->attr.comm)
atomic_dec(&nr_comm_events);
if (event->attr.namespaces)
@@ -8399,6 +8424,8 @@ static void perf_output_read(struct perf_output_handle *handle,
perf_output_read_one(handle, event, enabled, running);
}
+static struct unwind_work perf_unwind_work;
+
static void perf_bpf_to_sample_bid_offset(struct bpf_stack_build_id *bpf_bid)
{
struct perf_sample_build_id_offset *bid = (void *)bpf_bid;
@@ -8434,7 +8461,28 @@ static void perf_output_sample_build_id(struct perf_output_handle *handle,
PERF_RECORD_MISC_USER;
if (sample_bid) {
- if (is_user && has_user_ctx) {
+ u64 defer_cookie = 0;
+ bool defer_user = IS_ENABLED(CONFIG_UNWIND_USER) &&
+ is_user && has_user_ctx &&
+ event->attr.defer_callchain &&
+ (!(sample_type & PERF_SAMPLE_CALLCHAIN) ||
+ event->attr.exclude_callchain_user) &&
+ data->ip == instruction_pointer(task_pt_regs(current));
+
+ if (defer_user &&
+ unwind_deferred_request(&perf_unwind_work, &defer_cookie) < 0)
+ defer_cookie = 0;
+
+ if (defer_cookie) {
+ u64 deferred_bid[4] = {
+ 0,
+ PERF_CONTEXT_USER_DEFERRED,
+ defer_cookie,
+ data->ip,
+ };
+
+ perf_output_put(handle, deferred_bid);
+ } else if (is_user && has_user_ctx) {
if (callchain_bid && data->callchain_build_id &&
data->callchain_build_id->has_ip_bid) {
perf_output_put(handle,
@@ -8811,8 +8859,6 @@ static u64 perf_get_page_size(unsigned long addr)
static struct perf_callchain_entry __empty_callchain = { .nr = 0, };
-static struct unwind_work perf_unwind_work;
-
struct perf_callchain_entry *
perf_callchain(struct perf_event *event, struct pt_regs *regs)
{
@@ -10635,34 +10681,117 @@ void perf_event_bpf_event(struct bpf_prog *prog,
struct perf_callchain_deferred_event {
struct unwind_stacktrace *trace;
+ struct perf_callchain_build_id_entry *bid_entry;
struct {
struct perf_event_header header;
u64 cookie;
} event;
};
+static struct perf_callchain_build_id_entry *
+perf_resolve_build_id_offsets(struct unwind_stacktrace *trace)
+{
+ struct perf_callchain_build_id_entry *entry;
+ struct bpf_stack_build_id *bpf_bids;
+ u16 nr = trace->nr;
+ u32 max_size;
+
+ if (!nr || !current->mm ||
+ !atomic_read(&nr_defer_output_build_id_events))
+ return NULL;
+
+ if (!atomic_read(&nr_defer_output_callchain_build_id_events))
+ nr = 1;
+
+ entry = kzalloc(sizeof(*entry) + nr * sizeof(u64), GFP_KERNEL);
+ if (!entry)
+ return NULL;
+
+ bpf_bids = kcalloc(nr, sizeof(*bpf_bids), GFP_KERNEL);
+ if (!bpf_bids) {
+ kfree(entry);
+ return NULL;
+ }
+
+ for (u16 i = 0; i < nr; i++) {
+ entry->offsets[i] = trace->entries[i];
+ bpf_bids[i].ip = trace->entries[i];
+ }
+
+ stack_map_get_build_id_offset(bpf_bids, nr, true, true);
+
+ max_size = round_down(U16_MAX - sizeof(struct perf_event_header) -
+ 2 * sizeof(u64), sizeof(u64));
+ perf_callchain_encode_build_ids(entry, entry->offsets, nr, 0,
+ bpf_bids, nr, max_size);
+ kfree(bpf_bids);
+ return entry;
+}
+
static void perf_callchain_deferred_output(struct perf_event *event, void *data)
{
struct perf_callchain_deferred_event *deferred_event = data;
+ struct perf_callchain_build_id_entry *bid_entry = deferred_event->bid_entry;
struct perf_output_handle handle;
struct perf_sample_data sample;
u64 nr = deferred_event->trace->nr;
- u16 size, id_size, max_payload;
+ u32 max_offsets = 0, nr_build_ids = 0, nr_offsets = 0;
+ u32 raw_bids_len = 0, aligned_bids_len = 0;
+ bool use_bid;
+ u16 size, misc, id_size, max_payload;
int ret;
if (!event->attr.defer_output)
return;
+ use_bid = has_defer_output_build_id_offset(event);
+ if (use_bid && bid_entry) {
+ if (has_defer_output_callchain_build_id_offset(event))
+ max_offsets = bid_entry->nr_offsets;
+ else
+ max_offsets = min_t(u32, bid_entry->nr_offsets, 1);
+ }
+
id_size = event->attr.sample_id_all ? event->id_header_size : 0;
max_payload = round_down(U16_MAX - sizeof(deferred_event->event) -
sizeof(u64) - id_size, sizeof(u64));
- nr = min_t(u64, nr, max_payload / sizeof(u64));
- size = sizeof(deferred_event->event) + sizeof(u64) + nr * sizeof(u64);
+
+ if (use_bid) {
+ for (u32 i = 0; i < max_offsets; i++) {
+ u8 bid_idx = bid_entry->offsets[i] >> 56;
+ u32 next_nr_bids = nr_build_ids;
+ u32 next_raw_len = raw_bids_len;
+ u32 next_aligned_len;
+
+ if (bid_idx > 0 && bid_idx <= bid_entry->nr_build_ids &&
+ bid_idx > next_nr_bids) {
+ next_nr_bids = bid_idx;
+ next_raw_len = next_nr_bids *
+ (1 + BUILD_ID_SIZE_MAX);
+ }
+ next_aligned_len = ALIGN(next_raw_len, sizeof(u64));
+ if (next_aligned_len + (i + 1) * sizeof(u64) > max_payload)
+ break;
+ nr_build_ids = next_nr_bids;
+ raw_bids_len = next_raw_len;
+ aligned_bids_len = next_aligned_len;
+ nr_offsets = i + 1;
+ }
+ size = sizeof(deferred_event->event) + sizeof(u64) +
+ aligned_bids_len + nr_offsets * sizeof(u64);
+ } else {
+ nr = min_t(u64, nr, max_payload / sizeof(u64));
+ size = sizeof(deferred_event->event) + sizeof(u64) +
+ nr * sizeof(u64);
+ }
+
+ misc = PERF_RECORD_MISC_USER |
+ (use_bid ? PERF_RECORD_MISC_MMAP_BUILD_ID : 0);
/* XXX do we really need sample_id_all for this ??? */
perf_event_header__init(&deferred_event->event.header, &sample,
PERF_RECORD_CALLCHAIN_DEFERRED,
- PERF_RECORD_MISC_USER,
+ misc,
size,
event);
@@ -10672,11 +10801,27 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data)
return;
perf_output_put(&handle, deferred_event->event);
- perf_output_put(&handle, nr);
- for (u64 i = 0; i < nr; i++) {
- u64 entry = deferred_event->trace->entries[i];
+ if (use_bid) {
+ u64 pad_zero = 0;
+ u32 pad = aligned_bids_len - raw_bids_len;
+
+ perf_output_put(&handle, nr_build_ids);
+ perf_output_put(&handle, nr_offsets);
+ if (raw_bids_len)
+ __output_copy(&handle, bid_entry->build_ids, raw_bids_len);
+ if (pad)
+ __output_copy(&handle, &pad_zero, pad);
+ if (nr_offsets) {
+ __output_copy(&handle, bid_entry->offsets,
+ nr_offsets * sizeof(u64));
+ }
+ } else {
+ perf_output_put(&handle, nr);
+ for (u64 i = 0; i < nr; i++) {
+ u64 entry = deferred_event->trace->entries[i];
- perf_output_put(&handle, entry);
+ perf_output_put(&handle, entry);
+ }
}
perf_event__output_id_sample(event, &handle, &sample);
@@ -10686,14 +10831,18 @@ static void perf_callchain_deferred_output(struct perf_event *event, void *data)
static void perf_unwind_deferred_callback(struct unwind_work *work,
struct unwind_stacktrace *trace, u64 cookie)
{
+ struct perf_callchain_build_id_entry *bid_entry =
+ perf_resolve_build_id_offsets(trace);
struct perf_callchain_deferred_event deferred_event = {
.trace = trace,
+ .bid_entry = bid_entry,
.event = {
.cookie = cookie,
},
};
perf_iterate_sb(perf_callchain_deferred_output, &deferred_event, NULL);
+ kfree(bid_entry);
}
struct perf_text_poke_event {
@@ -13552,6 +13701,10 @@ static void account_event(struct perf_event *event)
atomic_inc(&nr_mmap_events);
if (event->attr.build_id)
atomic_inc(&nr_build_id_events);
+ if (has_defer_output_build_id_offset(event))
+ atomic_inc(&nr_defer_output_build_id_events);
+ if (has_defer_output_callchain_build_id_offset(event))
+ atomic_inc(&nr_defer_output_callchain_build_id_events);
if (event->attr.comm)
atomic_inc(&nr_comm_events);
if (event->attr.namespaces)
--
2.56.1.703.gf848ed807c-goog