Re: Lockdep circular dependency: btrfs_swap_activate() calls sysfs_notify()

[Christian Borntraeger]

Am 31.07.26 um 13:12 schrieb Qu Wenruo:
> 在 2026/7/31 20:27, Christian Borntraeger 写道:
> > One or more of the following files ( btrfs-swapon-kernfs-repro.sh ) violates IBM policy and all attachment(s) have been removed from the message.

Oh.


Lets try this:
#!/bin/bash
# Reproducer for the lockdep circular dependency
#
#   &root->kernfs_rwsem --> sb_pagefaults --> &ei->i_mmap_lock
#
# btrfs_swap_activate() calls btrfs_exclop_finish() -> sysfs_notify() ->
# kernfs_rwsem while holding the inode's i_mmap_lock for write, while
# kernfs_fop_readdir() holds kernfs_rwsem across filldir64() (-> mmap_lock)
# and a btrfs write fault goes mmap_lock -> sb_pagefaults -> i_mmap_lock.
#
#
# Must be run as root, on a kernel with CONFIG_PROVE_LOCKING and
# CONFIG_DEBUG_ATOMIC_SLEEP.  Run it on an unpatched kernel to see the splat,
# and on a patched one to confirm it is gone.
#
# Usage: ./btrfs-swapon-kernfs-repro.sh [existing-btrfs-mountpoint]
#
# With no argument a 512M loopback btrfs image is created under /var/tmp and
# removed again at the end.  With an argument, that existing single-device
# btrfs mount is used and only the test files inside it are removed.
#
# Exit codes: 0 = no splat (expected with the fix), 1 = reproduced,
#             2 = inconclusive (setup problem, or lockdep already off).

set -u

IMG=""
LOOP=""
MNT=""
OWN_FS=0
SWAPFILE=""
SWAP_ON=0
SWAPON_OK=0

msg()  { printf '[repro] %s\n' "$*"; }
fail() { printf '[repro] ERROR: %s\n' "$*" >&2; exit 2; }

cleanup() {
	set +e
	[ "$SWAP_ON" = 1 ] && { msg "swapoff $SWAPFILE"; swapoff "$SWAPFILE"; }
	if [ "$OWN_FS" = 1 ]; then
		[ -n "$MNT" ]  && umount "$MNT" 2>/dev/null && rmdir "$MNT"
		[ -n "$LOOP" ] && losetup -d "$LOOP" 2>/dev/null
		[ -n "$IMG" ]  && rm -f "$IMG"
	elif [ -n "$MNT" ]; then
		# Only ever the two files this script created, never a bare path.
		[ -n "$SWAPFILE" ] && rm -f "$SWAPFILE"
		rm -f "$MNT/lockdep-mmap.dat"
	fi
}
trap cleanup EXIT

# ---------------------------------------------------------------- checks ---

[ "$(id -u)" = 0 ] || fail "must be run as root"

for t in mkfs.btrfs losetup mkswap swapon swapoff chattr dmesg \
	 truncate stat find dd sync mount umount; do
	command -v "$t" >/dev/null || fail "missing tool: $t"
done

[ -e /proc/lockdep_stats ] || fail \
	"no /proc/lockdep_stats -- kernel lacks CONFIG_PROVE_LOCKING, nothing to see"

# A previous splat disables lockdep for the rest of the boot ("INFO: lockdep
# is turned off").  Nothing will be reported after that, so bail out early
# rather than report a bogus PASS.
if grep -qE '^ *debug_locks: *0' /proc/lockdep_stats; then
	fail "lockdep is already turned off (earlier splat this boot) -- reboot first"
fi

# Edge #1 of the cycle needs the might_fault() annotation in filldir64().
KCONF=""
for c in /proc/config.gz "/boot/config-$(uname -r)"; do
	[ -e "$c" ] && { KCONF="$c"; break; }
done
if [ -n "$KCONF" ]; then
	if [ "${KCONF##*.}" = gz ]; then RDCONF="zcat"; else RDCONF="cat"; fi
	$RDCONF "$KCONF" | grep -q '^CONFIG_DEBUG_ATOMIC_SLEEP=y' || msg \
		"WARNING: CONFIG_DEBUG_ATOMIC_SLEEP is not set -- filldir64() will not
         annotate mmap_lock and the cycle cannot be detected"
else
	msg "note: could not find the kernel config, not checking DEBUG_ATOMIC_SLEEP"
fi

MMAP_HELPER=""
if command -v python3 >/dev/null; then
	MMAP_HELPER=python3
elif command -v cc >/dev/null; then
	MMAP_HELPER=cc
else
	fail "need python3 or cc to do the mmap write fault"
fi

msg "kernel $(uname -r), lockdep active"

# Cursor for the verdict.  Taken before the first step, not just before
# swapon: if some earlier boot activity already registered the
# i_mmap_lock -> kernfs_rwsem edge, the cycle is completed by the mmap or
# the readdir step below instead, and the splat has to be caught there too.
DMESG_LINES=$(dmesg | wc -l)

# ------------------------------------------------------------ test btrfs ---

if [ $# -ge 1 ]; then
	MNT="$1"
	[ -d "$MNT" ] || fail "$MNT is not a directory"
	[ "$(stat -f -c %T "$MNT")" = btrfs ] || fail "$MNT is not btrfs"
	msg "using existing btrfs at $MNT"
else
	OWN_FS=1
	IMG=$(mktemp /var/tmp/btrfs-swapon-repro.XXXXXX.img) || fail "mktemp failed"
	MNT=$(mktemp -d /var/tmp/btrfs-swapon-repro.XXXXXX.mnt) || fail "mktemp failed"
	msg "creating 512M btrfs image $IMG"
	truncate -s 512M "$IMG"                     || fail "truncate failed"
	LOOP=$(losetup --find --show "$IMG")        || fail "losetup failed"
	mkfs.btrfs -q -f "$LOOP"                    || fail "mkfs.btrfs failed"
	# No compression: a compressed swap file is rejected by btrfs.
	mount -o compress=no "$LOOP" "$MNT"         || fail "mount failed"
	msg "mounted $LOOP on $MNT"
fi

# ------------------------------------ edges #2/#3: mmap write fault on btrfs --
# handle_mm_fault -> do_page_mkwrite -> btrfs_page_mkwrite
#   -> sb_start_pagefault()            [sb_pagefaults]
#   -> down_read(&BTRFS_I(inode)->i_mmap_lock)
# Both the do_fault (first touch is a write) and the do_wp_page (read first,
# then write) variants are exercised.

MMAPFILE="$MNT/lockdep-mmap.dat"
dd if=/dev/zero of="$MMAPFILE" bs=4096 count=64 status=none || fail "dd failed"
sync

msg "dirtying a btrfs file mapping (mmap_lock -> sb_pagefaults -> i_mmap_lock)"
if [ "$MMAP_HELPER" = python3 ]; then
	python3 - "$MMAPFILE" <<-'EOF' || fail "mmap helper failed"
		import mmap, os, sys
		fd = os.open(sys.argv[1], os.O_RDWR)
		m = mmap.mmap(fd, 4096 * 64)
		m[0:8] = b"lockdep!"          # write fault on an absent page (do_fault)
		_ = m[8192]                   # populate clean ...
		m[8192:8200] = b"lockdep!"    # ... then dirty it (do_wp_page)
		m.flush()
		m.close()
		os.close(fd)
	EOF
else
	HELPER=$(mktemp -d)
	cat > "$HELPER/mw.c" <<-'EOF'
		#include <fcntl.h>
		#include <stdio.h>
		#include <sys/mman.h>
		#include <unistd.h>
		int main(int argc, char **argv)
		{
			size_t len = 4096 * 64;
			char *p;
			int fd = open(argv[1], O_RDWR);

			if (fd < 0)
				return perror("open"), 1;
			p = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
			if (p == MAP_FAILED)
				return perror("mmap"), 1;
			p[0] = 'x';                  /* do_fault */
			(void)*(volatile char *)(p + 8192);
			p[8192] = 'x';               /* do_wp_page */
			msync(p, len, MS_SYNC);
			munmap(p, len);
			close(fd);
			return 0;
		}
	EOF
	cc -O2 -o "$HELPER/mw" "$HELPER/mw.c" || fail "compiling the mmap helper failed"
	"$HELPER/mw" "$MMAPFILE" || fail "mmap helper failed"
	rm -rf "$HELPER"
fi

# ------------------------------------------- edge #1: kernfs readdir + fault --
# kernfs_fop_readdir() holds kernfs_rwsem for read across the whole emit loop
# and calls filldir64() -> __might_fault() -> mmap_lock.  Any getdents on a
# sysfs directory registers it.

msg "reading sysfs directories (kernfs_rwsem -> mmap_lock)"
for d in /sys/fs/btrfs /sys/class/net /sys/block /sys/devices/system/cpu; do
	[ -d "$d" ] && ls -fU "$d" >/dev/null 2>&1
done
find /sys/fs/btrfs -maxdepth 3 >/dev/null 2>&1

# ------------------------------------------- edge #0: swapon on btrfs (bang) --
# btrfs_swap_activate() holds i_mmap_lock for write across
# btrfs_exclop_finish() -> sysfs_notify() -> kernfs_find_and_get_ns()
#   -> down_read(&root->kernfs_rwsem)
#
# The swap file has to satisfy every btrfs_swap_activate() precondition:
# NODATACOW and NODATASUM (both from chattr +C on a still empty file), not
# compressed, no holes, no inline extent, not shared.  fallocate() is not
# used on purpose -- the extents are written out for real.

SWAPFILE="$MNT/lockdep-swapfile"
msg "creating swap file $SWAPFILE"
rm -f "$SWAPFILE"
touch "$SWAPFILE"                 || fail "touch failed"
chattr +C "$SWAPFILE"             || fail "chattr +C failed (needs NODATACOW)"
chmod 600 "$SWAPFILE"
dd if=/dev/zero of="$SWAPFILE" bs=1M count=64 status=none || fail "dd failed"
sync
mkswap "$SWAPFILE" >/dev/null     || fail "mkswap failed"

msg "swapon -- this is the call that closes the cycle"
if swapon -p 0 "$SWAPFILE"; then
	SWAP_ON=1
	SWAPON_OK=1
	msg "swapon succeeded"
else
	msg "swapon FAILED -- check the btrfs_warn above for which precondition"
	msg "(the lockdep edge is taken on the error paths too, so keep reading)"
fi

sleep 1

# ------------------------------------------------------------------ verdict --

NEW=$(dmesg | tail -n +$((DMESG_LINES + 1)))

if printf '%s\n' "$NEW" | grep -q "possible circular locking dependency"; then
	printf '%s\n' "$NEW" | sed -n '/possible circular locking dependency/,$p'
	if printf '%s\n' "$NEW" | grep -q "kernfs_rwsem"; then
		msg "REPRODUCED: the expected cycle, kernfs_rwsem is in the chain"
		exit 1
	fi
	msg "INCONCLUSIVE: a circular dependency was reported, but without"
	msg "kernfs_rwsem -- that is a different bug, compare against more.txt"
	exit 2
fi

msg "no lockdep splat"
if grep -qE '^ *debug_locks: *0' /proc/lockdep_stats; then
	msg "INCONCLUSIVE: lockdep turned itself off during the run"
	exit 2
fi
if [ "$SWAPON_OK" = 0 ]; then
	msg "INCONCLUSIVE: swapon did not get far enough, fix that first"
	exit 2
fi
msg "PASS: the i_mmap_lock -> kernfs_rwsem edge was not taken"
msg "(expected on a kernel carrying btrfs-swapon-kernfs-fix.patch)"
exit 0