[PATCH v13 15/21] perf intel-tpebs: Fix race in tpebs_stop() and tpebs_cmd.out fd handling

From: Ian Rogers

Date: Fri Nov 13 2026 - 12:04:31 EST


intel-tpebs has two lifecycle bugs around the background 'perf record'
process and sample reader thread:
1. tpebs_cmd.out is read without tpebs_mtx and closed by both
perf_session__delete() in __sample_reader() and tpebs_stop() (or
leaked if perf_session__new() fails).
2. tpebs_stop() reads tpebs_reader_thread after unlocking tpebs_mtx, and
tpebs_send_record_cmd() drops tpebs_mtx while writing undelimited
control strings and polling ack_fd, allowing concurrent open/close
calls to coalesce commands or race to close pipes, join the reader
thread, and reap tpebs_cmd twice.

Fix (1) by passing tpebs_cmd.out to __sample_reader() and resetting
tpebs_cmd.out = -1 once ownership is transferred (and on start_command()
failure). Fix (2) by reading tpebs_reader_thread under tpebs_mtx,
newline-terminating control commands, serializing tpebs_send_record_cmd()
with teardown via a condition variable and generation counter, and
ignoring late samples in process_sample_event() while stopping. Add
explicit header includes used by the file.

Fixes: 8db5cabcf1b6 ("perf stat: Fork and launch 'perf record' when 'perf stat' needs to get retire latency value for a metric.")
Fixes: bded0398ac19 ("perf tpebs: Fix concurrent stop races and PID reuse hazards in tpebs_stop")
Signed-off-by: Ian Rogers <irogers@xxxxxxxxxx>
---
tools/perf/util/intel-tpebs.c | 141 ++++++++++++++++++++++++++++------
1 file changed, 119 insertions(+), 22 deletions(-)

diff --git a/tools/perf/util/intel-tpebs.c b/tools/perf/util/intel-tpebs.c
index 7c0b8b15bd10..5cb090b27fbd 100644
--- a/tools/perf/util/intel-tpebs.c
+++ b/tools/perf/util/intel-tpebs.c
@@ -27,6 +27,12 @@
#include <errno.h>
#include <poll.h>
#include <math.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>

#define PERF_DATA "-"

@@ -37,7 +43,11 @@ static pthread_t tpebs_reader_thread;
static struct child_process tpebs_cmd;
static int control_fd[2], ack_fd[2];
static struct mutex tpebs_mtx;
+static struct cond tpebs_cmd_cond = { .cond = PTHREAD_COND_INITIALIZER };
static bool tpebs_stopping;
+static bool tpebs_enabled;
+static bool tpebs_cmd_in_flight;
+static uint64_t tpebs_gen;

struct tpebs_retire_lat {
struct list_head nd;
@@ -120,10 +130,15 @@ static int evsel__tpebs_start_perf_record(struct evsel *evsel)
tpebs_cmd.out = -1;
ret = start_command(&tpebs_cmd);
zfree(&tpebs_cmd.argv);
+ if (ret) {
+ tpebs_cmd.pid = 0;
+ tpebs_cmd.out = -1;
+ return ret;
+ }
list_for_each_entry(t, &tpebs_results, nd)
t->started = true;

- return ret;
+ return 0;
}

static bool is_child_pid(pid_t parent, pid_t child)
@@ -195,8 +210,8 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused,
struct tpebs_retire_lat *t;

mutex_lock(tpebs_mtx_get());
- if (tpebs_cmd.pid == 0) {
- /* Record has terminated. */
+ if (tpebs_stopping || tpebs_cmd.pid == 0) {
+ /* Record is stopping or has terminated. */
mutex_unlock(tpebs_mtx_get());
return 0;
}
@@ -220,13 +235,13 @@ static int process_sample_event(const struct perf_tool *tool __maybe_unused,
return 0;
}

-static void *__sample_reader(void *arg __maybe_unused)
+static void *__sample_reader(void *arg)
{
struct perf_session *session;
struct perf_data data = {
.mode = PERF_DATA_MODE_READ,
.path = PERF_DATA,
- .file.fd = tpebs_cmd.out,
+ .file.fd = (int)(intptr_t)arg,
};
struct perf_tool tool;

@@ -236,8 +251,10 @@ static void *__sample_reader(void *arg __maybe_unused)
tool.attr = perf_event__process_attr;

session = perf_session__new(&data, &tool);
- if (IS_ERR(session))
+ if (IS_ERR(session)) {
+ close(data.file.fd);
return NULL;
+ }
perf_session__process_events(session);
perf_session__delete(session);

@@ -247,35 +264,49 @@ static void *__sample_reader(void *arg __maybe_unused)
static int tpebs_send_record_cmd(const char *msg) __must_hold(tpebs_mtx_get())
{
struct pollfd pollfd = { .events = POLLIN, };
- int ret, len, retries = 0;
+ bool is_stop = !strcmp(msg, EVLIST_CTL_CMD_STOP_TAG);
+ int ret, len, retries = 0, ctrl_fd, resp_fd;
+ char cmd_buf[EVLIST_CTL_CMD_MAX_LEN];
char ack_buf[8];

+ while (tpebs_cmd_in_flight) {
+ if (!is_stop && tpebs_stopping)
+ return -EBUSY;
+ cond_wait(&tpebs_cmd_cond, tpebs_mtx_get());
+ }
+ if (!is_stop && tpebs_stopping)
+ return -EBUSY;
+
/* Check if the command exited before the send, done with the lock held. */
if (tpebs_cmd.pid == 0)
return 0;

+ ctrl_fd = control_fd[1];
+ resp_fd = ack_fd[0];
+ tpebs_cmd_in_flight = true;
+
/*
* Let go of the lock while sending/receiving as blocking can starve the
* sample reading thread.
*/
mutex_unlock(tpebs_mtx_get());

- /* Send perf record command.*/
- len = strlen(msg);
- ret = write(control_fd[1], msg, len);
+ /* Send perf record command. */
+ len = scnprintf(cmd_buf, sizeof(cmd_buf), "%s\n", msg);
+ ret = write(ctrl_fd, cmd_buf, len);
if (ret != len) {
pr_err("perf record control write control message '%s' failed\n", msg);
ret = -EPIPE;
goto out;
}

- if (!strcmp(msg, EVLIST_CTL_CMD_STOP_TAG)) {
+ if (is_stop) {
ret = 0;
goto out;
}

/* Wait for an ack. */
- pollfd.fd = ack_fd[0];
+ pollfd.fd = resp_fd;

/*
* We need this poll to ensure the ack_fd PIPE will not hang
@@ -284,7 +315,17 @@ static int tpebs_send_record_cmd(const char *msg) __must_hold(tpebs_mtx_get())
*/
again:
if (!poll(&pollfd, 1, 500)) {
- if (check_if_command_finished(&tpebs_cmd)) {
+ bool stopping, finished;
+
+ mutex_lock(tpebs_mtx_get());
+ stopping = tpebs_stopping;
+ finished = !stopping && check_if_command_finished(&tpebs_cmd);
+ mutex_unlock(tpebs_mtx_get());
+ if (stopping) {
+ ret = -EBUSY;
+ goto out;
+ }
+ if (finished) {
ret = 0;
goto out;
}
@@ -297,7 +338,17 @@ static int tpebs_send_record_cmd(const char *msg) __must_hold(tpebs_mtx_get())
}

if (!(pollfd.revents & POLLIN)) {
- if (check_if_command_finished(&tpebs_cmd)) {
+ bool stopping, finished;
+
+ mutex_lock(tpebs_mtx_get());
+ stopping = tpebs_stopping;
+ finished = !stopping && check_if_command_finished(&tpebs_cmd);
+ mutex_unlock(tpebs_mtx_get());
+ if (stopping) {
+ ret = -EBUSY;
+ goto out;
+ }
+ if (finished) {
ret = 0;
goto out;
}
@@ -307,7 +358,7 @@ static int tpebs_send_record_cmd(const char *msg) __must_hold(tpebs_mtx_get())
goto out;
}

- ret = read(ack_fd[0], ack_buf, sizeof(ack_buf));
+ ret = read(resp_fd, ack_buf, sizeof(ack_buf));
if (ret > 0)
ret = strcmp(ack_buf, EVLIST_CTL_CMD_ACK_TAG);
else
@@ -315,6 +366,8 @@ static int tpebs_send_record_cmd(const char *msg) __must_hold(tpebs_mtx_get())
out:
/* Re-take lock as expected by caller. */
mutex_lock(tpebs_mtx_get());
+ tpebs_cmd_in_flight = false;
+ cond_broadcast(&tpebs_cmd_cond);
return ret;
}

@@ -331,12 +384,23 @@ static int tpebs_stop(void) __must_hold(tpebs_mtx_get())
/* Like tpebs_start, we should only run tpebs_end once. */
if (tpebs_cmd.pid != 0) {
pid_t actual_pid = tpebs_cmd.pid;
+ bool enabled = tpebs_enabled;
+ pthread_t reader_thread;

tpebs_stopping = true;
- tpebs_send_record_cmd(EVLIST_CTL_CMD_STOP_TAG);
+ tpebs_enabled = false;
+ tpebs_gen++;
+ cond_broadcast(&tpebs_cmd_cond);
+ if (tpebs_send_record_cmd(EVLIST_CTL_CMD_STOP_TAG) || !enabled)
+ kill(actual_pid, SIGTERM);
+ /*
+ * tpebs_results is empty when stopping; clear pid before
+ * releasing tpebs_mtx to join the reader thread.
+ */
tpebs_cmd.pid = 0;
+ reader_thread = tpebs_reader_thread;
mutex_unlock(tpebs_mtx_get());
- pthread_join(tpebs_reader_thread, NULL);
+ pthread_join(reader_thread, NULL);
mutex_lock(tpebs_mtx_get());
if (control_fd[0] >= 0) {
close(control_fd[0]);
@@ -515,6 +579,8 @@ int evsel__tpebs_open(struct evsel *evsel)
int ret;
bool tpebs_empty;
bool started_process = false;
+ bool started_thread = false;
+ uint64_t open_gen = 0;

/* We should only run tpebs_start when tpebs_recording is enabled. */
if (!tpebs_recording)
@@ -550,6 +616,8 @@ int evsel__tpebs_open(struct evsel *evsel)
tpebs_empty = list_empty(&tpebs_results);
if (!tpebs_empty) {
started_process = true;
+ tpebs_enabled = false;
+ open_gen = ++tpebs_gen;
/*Create control and ack fd for --control*/
if (pipe(control_fd) < 0) {
pr_err("tpebs: Failed to create control fifo");
@@ -567,13 +635,22 @@ int evsel__tpebs_open(struct evsel *evsel)
goto out;

if (pthread_create(&tpebs_reader_thread, /*attr=*/NULL, __sample_reader,
- /*arg=*/NULL)) {
+ (void *)(intptr_t)tpebs_cmd.out)) {
kill(tpebs_cmd.pid, SIGTERM);
pr_err("Could not create thread to process sample data.\n");
ret = -1;
goto out;
}
+ /* __sample_reader now owns and closes the output fd. */
+ tpebs_cmd.out = -1;
+ started_thread = true;
ret = tpebs_send_record_cmd(EVLIST_CTL_CMD_ENABLE_TAG);
+ if (!ret) {
+ if (tpebs_gen != open_gen)
+ ret = -EBUSY;
+ else
+ tpebs_enabled = true;
+ }
}
out:
if (ret) {
@@ -584,11 +661,25 @@ int evsel__tpebs_open(struct evsel *evsel)
tpebs_retire_lat__delete(t);
}

- if (started_process) {
- if (tpebs_cmd.pid > 0) {
- kill(tpebs_cmd.pid, SIGTERM);
- finish_command(&tpebs_cmd);
+ if (started_process && tpebs_gen == open_gen) {
+ pid_t actual_pid = tpebs_cmd.pid;
+
+ tpebs_stopping = true;
+ tpebs_enabled = false;
+ tpebs_gen++;
+ cond_broadcast(&tpebs_cmd_cond);
+ if (actual_pid > 0) {
+ kill(actual_pid, SIGTERM);
+ while (tpebs_cmd_in_flight)
+ cond_wait(&tpebs_cmd_cond, tpebs_mtx_get());
tpebs_cmd.pid = 0;
+ if (started_thread) {
+ pthread_t reader_thread = tpebs_reader_thread;
+
+ mutex_unlock(tpebs_mtx_get());
+ pthread_join(reader_thread, NULL);
+ mutex_lock(tpebs_mtx_get());
+ }
}
if (tpebs_cmd.out >= 0) {
close(tpebs_cmd.out);
@@ -610,6 +701,12 @@ int evsel__tpebs_open(struct evsel *evsel)
close(ack_fd[1]);
ack_fd[1] = -1;
}
+ if (actual_pid > 0) {
+ tpebs_cmd.pid = actual_pid;
+ finish_command(&tpebs_cmd);
+ tpebs_cmd.pid = 0;
+ }
+ tpebs_stopping = false;
}
}
mutex_unlock(tpebs_mtx_get());
--
2.56.1.703.gf848ed807c-goog