[PATCH 1/4] selftests: ublk: propagate recovery worker failures

[shao.mingyin]

From: Shao Mingyin <shao.mingyin@xxxxxxxxxx>

The recovery tests update ERR_CODE in background subshells and use wait
without PIDs. A worker can report a failure while the parent still prints
PASS and exits with zero. Calling _show_result() from a failed worker also
removes shared backing files while other workers may still be using them.

Wait for each recorded worker PID and aggregate its status in the parent,
keeping failures ahead of skips and preserving errors across batches.
Return worker and device-creation errors without reporting or cleaning up
from the child, so the parent can finish waiting before shared cleanup.

Fixes: 57e13a2e8cd2 ("selftests: ublk: support user recovery")
Fixes: 533c87e2ed74 ("selftests: ublk: add test for UBLK_F_QUIESCE")
Fixes: 63276182c513 ("selftests: ublk: add user copy test cases")
Reviewed-by: Ming Lei <tom.leiming@xxxxxxxxx>
Signed-off-by: Shao Mingyin <shao.mingyin@xxxxxxxxxx>
---
v2:
  - simplify the worker wait loop, as requested by Ming Lei.
  - the simplified loop is exercised by the real-kernel recovery runs
    described in the stress-worker patch of this series.
v1: https://lore.kernel.org/all/202611092358134332uegNg75loROSwL53Em6A@xxxxxxxxxx/

Tested on native x86_64 Bash 5.2 with the recovery scripts and common
helpers from the unpatched and patched trees. Device and I/O operations
were replaced by test-only stubs. Single-worker failures at every launch
position, multiple failures, skips, signal termination and device-creation
errors no longer produce a final PASS or a zero exit status. Successful
runs still pass, and cleanup follows all worker exits. Prerequisite skips
still exit before preparation. The 71 scenarios passed their expected
checks on each tree. This does not exercise real ublk devices or I/O.

 tools/testing/selftests/ublk/test_common.sh   | 20 +++++++++++++++++--
 .../testing/selftests/ublk/test_recover_01.sh | 20 +++++++++++++++----
 .../testing/selftests/ublk/test_recover_02.sh | 20 +++++++++++++++----
 .../testing/selftests/ublk/test_recover_03.sh | 20 +++++++++++++++----
 .../testing/selftests/ublk/test_recover_04.sh | 14 ++++++++++---
 5 files changed, 77 insertions(+), 17 deletions(-)

diff --git a/tools/testing/selftests/ublk/test_common.sh b/tools/testing/selftests/ublk/test_common.sh
index af2ea4f..367df2d 100755
--- a/tools/testing/selftests/ublk/test_common.sh
+++ b/tools/testing/selftests/ublk/test_common.sh
@@ -379,6 +379,23 @@ run_io_and_kill_daemon()
 	fi
 }
 
+# Collect recovery workers in the parent before removing shared backing files.
+_wait_recovery_tests()
+{
+	local pid
+	local ret
+
+	for pid in "$@"; do
+		wait "$pid" && continue
+		ret=$?
+
+		# A failure takes precedence over a skipped worker.
+		if [ "$ERR_CODE" -eq 0 ] || [ "$ERR_CODE" -eq "$UBLK_SKIP_CODE" ]; then
+			ERR_CODE=$ret
+		fi
+	done
+}
+
 run_io_and_recover()
 {
 	local size=$1
@@ -387,8 +404,7 @@ run_io_and_recover()
 	local dev_id
 
 	shift 2
-	dev_id=$(_add_ublk_dev "$@")
-	_check_add_dev "$TID" $?
+	dev_id=$(_add_ublk_dev "$@") || return $?
 
 	fio --name=job1 --filename=/dev/ublkb"${dev_id}" --ioengine=libaio \
 		--rw=randread --iodepth=256 --size="${size}" --numjobs=4 \
diff --git a/tools/testing/selftests/ublk/test_recover_01.sh b/tools/testing/selftests/ublk/test_recover_01.sh
index 1cddc23..950928a 100755
--- a/tools/testing/selftests/ublk/test_recover_01.sh
+++ b/tools/testing/selftests/ublk/test_recover_01.sh
@@ -11,8 +11,8 @@ ublk_run_recover_test()
 	ERR_CODE=$?
 	if [ ${ERR_CODE} -ne 0 ]; then
 		echo "$TID failure: $*"
-		_show_result $TID $ERR_CODE
 	fi
+	return "$ERR_CODE"
 }
 
 if ! _have_program fio; then
@@ -25,20 +25,32 @@ _create_backfile 0 256M
 _create_backfile 1 128M
 _create_backfile 2 128M
 
+pids=()
 ublk_run_recover_test -t null -q 2 -r 1 -b &
+pids+=("$!")
 ublk_run_recover_test -t loop -q 2 -r 1 -b "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_recover_test -t stripe -q 2 -r 1 -b "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
+pids=()
 ublk_run_recover_test -t null -q 2 -r 1 &
+pids+=("$!")
 ublk_run_recover_test -t loop -q 2 -r 1 "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_recover_test -t stripe -q 2 -r 1 "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
+pids=()
 ublk_run_recover_test -t null -q 2 -r 1 -i 1 &
+pids+=("$!")
 ublk_run_recover_test -t loop -q 2 -r 1 -i 1 "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_recover_test -t stripe -q 2 -r 1 -i 1 "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
 _cleanup_test
 _show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_recover_02.sh b/tools/testing/selftests/ublk/test_recover_02.sh
index 9c3f481..beeeb35 100755
--- a/tools/testing/selftests/ublk/test_recover_02.sh
+++ b/tools/testing/selftests/ublk/test_recover_02.sh
@@ -11,8 +11,8 @@ ublk_run_recover_test()
 	ERR_CODE=$?
 	if [ ${ERR_CODE} -ne 0 ]; then
 		echo "$TID failure: $*"
-		_show_result $TID $ERR_CODE
 	fi
+	return "$ERR_CODE"
 }
 
 if ! _have_program fio; then
@@ -29,20 +29,32 @@ _create_backfile 0 256M
 _create_backfile 1 128M
 _create_backfile 2 128M
 
+pids=()
 ublk_run_recover_test -t null -q 2 -r 1 -z -b &
+pids+=("$!")
 ublk_run_recover_test -t loop -q 2 -r 1 -z -b "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_recover_test -t stripe -q 2 -r 1 -z -b "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
+pids=()
 ublk_run_recover_test -t null -q 2 -r 1 -z &
+pids+=("$!")
 ublk_run_recover_test -t loop -q 2 -r 1 -z "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_recover_test -t stripe -q 2 -r 1 -z "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
+pids=()
 ublk_run_recover_test -t null -q 2 -r 1 -z -i 1 &
+pids+=("$!")
 ublk_run_recover_test -t loop -q 2 -r 1 -z -i 1 "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_recover_test -t stripe -q 2 -r 1 -z -i 1 "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
 _cleanup_test
 _show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_recover_03.sh b/tools/testing/selftests/ublk/test_recover_03.sh
index 92f4012..0c6b312 100755
--- a/tools/testing/selftests/ublk/test_recover_03.sh
+++ b/tools/testing/selftests/ublk/test_recover_03.sh
@@ -11,8 +11,8 @@ ublk_run_quiesce_recover()
 	ERR_CODE=$?
 	if [ ${ERR_CODE} -ne 0 ]; then
 		echo "$TID failure: $*"
-		_show_result $TID $ERR_CODE
 	fi
+	return "$ERR_CODE"
 }
 
 if ! _have_feature "QUIESCE"; then
@@ -29,20 +29,32 @@ _create_backfile 0 256M
 _create_backfile 1 128M
 _create_backfile 2 128M
 
+pids=()
 ublk_run_quiesce_recover -t null -q 2 -r 1 -b &
+pids+=("$!")
 ublk_run_quiesce_recover -t loop -q 2 -r 1 -b "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_quiesce_recover -t stripe -q 2 -r 1 -b "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
+pids=()
 ublk_run_quiesce_recover -t null -q 2 -r 1 &
+pids+=("$!")
 ublk_run_quiesce_recover -t loop -q 2 -r 1 "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_quiesce_recover -t stripe -q 2 -r 1 "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
+pids=()
 ublk_run_quiesce_recover -t null -q 2 -r 1 -i 1 &
+pids+=("$!")
 ublk_run_quiesce_recover -t loop -q 2 -r 1 -i 1 "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_quiesce_recover -t stripe -q 2 -r 1 -i 1 "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
 _cleanup_test
 _show_result $TID $ERR_CODE
diff --git a/tools/testing/selftests/ublk/test_recover_04.sh b/tools/testing/selftests/ublk/test_recover_04.sh
index 4c83c18..232ae7d 100755
--- a/tools/testing/selftests/ublk/test_recover_04.sh
+++ b/tools/testing/selftests/ublk/test_recover_04.sh
@@ -11,8 +11,8 @@ ublk_run_recover_test()
 	ERR_CODE=$?
 	if [ ${ERR_CODE} -ne 0 ]; then
 		echo "$TID failure: $*"
-		_show_result $TID $ERR_CODE
 	fi
+	return "$ERR_CODE"
 }
 
 if ! _have_program fio; then
@@ -25,15 +25,23 @@ _create_backfile 0 256M
 _create_backfile 1 128M
 _create_backfile 2 128M
 
+pids=()
 ublk_run_recover_test -t null -q 2 -r 1 -u &
+pids+=("$!")
 ublk_run_recover_test -t loop -q 2 -r 1 -u "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_recover_test -t stripe -q 2 -r 1 -u "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
+pids=()
 ublk_run_recover_test -t null -q 2 -r 1 -u -i 1 &
+pids+=("$!")
 ublk_run_recover_test -t loop -q 2 -r 1 -u -i 1 "${UBLK_BACKFILES[0]}" &
+pids+=("$!")
 ublk_run_recover_test -t stripe -q 2 -r 1 -u -i 1 "${UBLK_BACKFILES[1]}" "${UBLK_BACKFILES[2]}" &
-wait
+pids+=("$!")
+_wait_recovery_tests "${pids[@]}"
 
 _cleanup_test
 _show_result $TID $ERR_CODE