[PATCH v3 5/9] perf test: Deflake failed-syscalls Python shell tests under load

[Ian Rogers]

When running 'perf test' in parallel under heavy system load, a one-shot
'perf record -e ... -- ls /does_not_exist' without '-B -N --no-bpf-event'
can exit in under 300 microseconds before 'ls's PERF_RECORD_COMM and
ENOENT sys_exit events are captured, while also contending on ~/.debug
build-id caching and BPF event synthesis.

Pass '-B -N --no-bpf-event' to 'perf record', sleep 0.05s in a subshell
after 'ls' so its events are flushed before the parent subshell exits,
and wrap the record and check steps in a bounded retry loop (up to 5
attempts).

Fixes: b76c43d09b06 ("perf python: Port failed-syscalls-by-pid to perf module")
Fixes: 4e3fe6987cba ("perf python: Port failed-syscalls from Perl to perf module")
Assisted-by: Antigravity:gemini-3.1-pro
Signed-off-by: Ian Rogers <irogers@xxxxxxxxxx>
---
 .../test_failed_syscalls_by_pid_python.sh     | 68 +++++++++++--------
 .../shell/test_failed_syscalls_python.sh      | 54 +++++++++------
 2 files changed, 71 insertions(+), 51 deletions(-)

diff --git a/tools/perf/tests/shell/test_failed_syscalls_by_pid_python.sh b/tools/perf/tests/shell/test_failed_syscalls_by_pid_python.sh
index 215372d4e1c5..070a7ef6b8ad 100755
--- a/tools/perf/tests/shell/test_failed_syscalls_by_pid_python.sh
+++ b/tools/perf/tests/shell/test_failed_syscalls_by_pid_python.sh
@@ -41,8 +41,10 @@ test_file_mode() {
 	echo "Testing failed-syscalls-by-pid.py..."
 
 	# Check if syscalls:sys_exit is supported/readable
-	if ! perf record -e syscalls:sys_exit -o /dev/null -- true >/dev/null 2>&1; then
-		if ! perf record -e raw_syscalls:sys_exit -o /dev/null -- true >/dev/null 2>&1; then
+	if ! perf record -B -N --no-bpf-event -e syscalls:sys_exit \
+		-o /dev/null -- true >/dev/null 2>&1; then
+		if ! perf record -B -N --no-bpf-event -e raw_syscalls:sys_exit \
+			-o /dev/null -- true >/dev/null 2>&1; then
 			echo "Skipping test, no syscalls:sys_exit or raw_syscalls:sys_exit event"
 			exit 2
 		else
@@ -52,41 +54,49 @@ test_file_mode() {
 		EVENT="syscalls:sys_exit"
 	fi
 
-	# Generate some events by running a command that should fail at least some syscall
-	# (e.g. failing stat on non-existent file).
-	# Using '|| true' because 'perf record' returns the exit code of 'ls',
-	# which fails with ENOENT
-	perf record -e "${EVENT}" -o "${temp_data}" -- ls /does_not_exist >/dev/null 2>&1 || true
+	# Generate some events by running a command that fails a syscall
+	# (e.g. failing stat on non-existent file), sleeping briefly in the
+	# subshell so ls's PERF_RECORD_COMM and sys_exit events are flushed.
+	passed=0
+	for _ in 1 2 3 4 5; do
+		rm -f "${temp_data}" "${temp_out}" "${temp_out}.comm" "${temp_out}.pid"
+		perf record -B -N --no-bpf-event -e "${EVENT}" -o "${temp_data}" \
+			-- sh -c "ls /does_not_exist 2>/dev/null; sleep 0.05 || true" \
+			>/dev/null 2>&1 || true
+		if [ ! -s "${temp_data}" ]; then
+			continue
+		fi
+
+		if perf script failed-syscalls-by-pid -i "${temp_data}" > "${temp_out}" && \
+		   grep -n -q "err = ENOENT" "${temp_out}" && \
+		   perf script failed-syscalls-by-pid -i "${temp_data}" \
+			"ls" > "${temp_out}.comm" && \
+		   grep -q "err = ENOENT" "${temp_out}.comm"; then
+			ls_pid=$(sed -n 's/^ls \[\([0-9][0-9]*\)\].*/\1/p' \
+				"${temp_out}.comm" | head -n 1)
+			if [ -n "${ls_pid}" ] && \
+			   perf script failed-syscalls-by-pid -i "${temp_data}" \
+				"${ls_pid}" > "${temp_out}.pid" && \
+			   grep -q "err = ENOENT" "${temp_out}.pid"; then
+				passed=1
+				break
+			fi
+		fi
+	done
+
 	if [ ! -s "${temp_data}" ]; then
 		echo "Skipping test, perf record failed to create data"
 		exit 2
 	fi
 
-	# Run the script and check output
-	if ! perf script failed-syscalls-by-pid -i "${temp_data}" > "${temp_out}"; then
+	if [ "$passed" -eq 0 ]; then
 		echo "failed-syscalls-by-pid test failed."
-		err=1
-	elif ! grep -n -q "err = ENOENT" "${temp_out}"; then
-		echo "Failed to find expected failed syscalls"
-		cat "${temp_out}"
-		err=1
-	elif ! perf script failed-syscalls-by-pid -i "${temp_data}" "ls" > "${temp_out}.comm" || \
-	     ! grep -q "err = ENOENT" "${temp_out}.comm"; then
-		echo "failed-syscalls-by-pid comm filter test failed."
-		cat "${temp_out}.comm"
+		cat "${temp_out}" 2>/dev/null || true
+		cat "${temp_out}.comm" 2>/dev/null || true
+		cat "${temp_out}.pid" 2>/dev/null || true
 		err=1
 	else
-		ls_pid=$(sed -n 's/^ls \[\([0-9][0-9]*\)\].*/\1/p' "${temp_out}.comm" | head -n 1)
-		if [ -z "${ls_pid}" ] || \
-		   ! perf script failed-syscalls-by-pid -i "${temp_data}" \
-			"${ls_pid}" > "${temp_out}.pid" || \
-		   ! grep -q "err = ENOENT" "${temp_out}.pid"; then
-			echo "failed-syscalls-by-pid PID filter test failed."
-			cat "${temp_out}.pid" 2>/dev/null || true
-			err=1
-		else
-			echo "failed-syscalls-by-pid test passed."
-		fi
+		echo "failed-syscalls-by-pid test passed."
 	fi
 	rm -f "${temp_out}" "${temp_out}.comm" "${temp_out}.pid"
 }
diff --git a/tools/perf/tests/shell/test_failed_syscalls_python.sh b/tools/perf/tests/shell/test_failed_syscalls_python.sh
index 861c2ba71c0a..9aaf05883c50 100755
--- a/tools/perf/tests/shell/test_failed_syscalls_python.sh
+++ b/tools/perf/tests/shell/test_failed_syscalls_python.sh
@@ -27,22 +27,22 @@ if [ ! -f "$script_path" ]; then
 fi
 
 err=0
-temp_data=""
-temp_out=""
+temp_dir=$(mktemp -d /tmp/perf-failed-syscalls-XXXXXX)
+temp_data="${temp_dir}/perf.data"
+temp_out="${temp_dir}/perf.out"
 
 cleanup() {
-	rm -f "${temp_data}" "${temp_out}"
+	rm -rf "${temp_dir}"
 }
 trap 'cleanup' EXIT TERM INT
 
-temp_data=$(mktemp /tmp/perf.data.XXXXXX)
-temp_out=$(mktemp /tmp/perf.out.XXXXXX)
-
 echo "Testing failed-syscalls.py..."
 
 # Check if sys_exit event can be recorded
-if ! perf record -e raw_syscalls:sys_exit -o /dev/null -- true >/dev/null 2>&1; then
-	if ! perf record -e syscalls:sys_exit -o /dev/null -- true >/dev/null 2>&1; then
+if ! perf record -B -N --no-bpf-event -e raw_syscalls:sys_exit \
+	-o /dev/null -- true >/dev/null 2>&1; then
+	if ! perf record -B -N --no-bpf-event -e syscalls:sys_exit \
+		-o /dev/null -- true >/dev/null 2>&1; then
 		echo "Skipping test, no permission or support for sys_exit event"
 		exit 2
 	else
@@ -52,28 +52,38 @@ else
 	EVENT="raw_syscalls:sys_exit"
 fi
 
-# Run perf record with a command that fails a syscall (ls non-existent file).
-# ls exits with non-zero, so perf record returns non-zero exit code of the workload.
-perf record -e "${EVENT}" -o "${temp_data}" \
-	-- ls /nonexistent_file_for_test >/dev/null 2>&1 || true
+# Run perf record with a command that fails a syscall (ls non-existent file),
+# sleeping briefly in the subshell so ls's PERF_RECORD_COMM and sys_exit events are flushed.
+passed=0
+for _ in 1 2 3 4 5; do
+	rm -f "${temp_data}" "${temp_out}"
+	perf record -B -N --no-bpf-event -e "${EVENT}" -o "${temp_data}" \
+		-- sh -c "ls /nonexistent_file_for_test 2>/dev/null; sleep 0.05 || true" \
+		>/dev/null 2>&1 || true
+
+	if [ ! -s "${temp_data}" ]; then
+		continue
+	fi
+
+	# Check that the script executes
+	if perf script failed-syscalls -i "${temp_data}" > "${temp_out}" && \
+	   grep -q "failed syscalls by comm" "${temp_out}" && \
+	   grep -Eq '^ls[[:space:]]+[0-9]+' "${temp_out}"; then
+		passed=1
+		break
+	fi
+done
 
 if [ ! -s "${temp_data}" ]; then
 	echo "Skipping test, perf record failed to create data"
 	exit 2
 fi
 
-# Check that the script executes
-if ! perf script failed-syscalls -i "${temp_data}" > "${temp_out}"; then
-	echo "failed-syscalls.py test failed"
+if [ "$passed" -eq 0 ]; then
+	echo "Failed to find the metrics table header or expected error"
 	err=1
 else
-	if ! grep -q "failed syscalls by comm" "${temp_out}" || \
-		! grep -Eq '^ls[[:space:]]+[0-9]+' "${temp_out}"; then
-		echo "Failed to find the metrics table header or expected error"
-		err=1
-	else
-		echo "failed-syscalls test passed."
-	fi
+	echo "failed-syscalls test passed."
 fi
 rm -f "${temp_out}"
 
-- 
2.56.0.rc1.315.gc6ed9934b7-goog