Re: [PATCH] drm/amdgpu: do not enter fs_reclaim under notifier_lock in lockdep training

[Mikhail Gavrilov]

Makes sense, thanks. I won't respin this one then.

Vitaly, for the reorder, here is a deterministic reproducer so you can confirm
the splat on your side without a round-trip. It arms an mmu_interval_notifier
via GEM_USERPTR over anonymous memory, then forces reclaim of that exact range
with madvise(MADV_PAGEOUT), so amdgpu_hmm_invalidate_gfx() takes notifier_lock
under fs_reclaim in the calling thread. Needs CONFIG_PROVE_LOCKING and a fresh
boot; build/run notes are in the header. Happy to give Tested-by once you post.

// SPDX-License-Identifier: MIT
/*
 * amdgpu-notifier-reclaim-repro.c
 *
 * Deterministic reproducer for the false circular-locking-dependency splat
 * produced by drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c.
 *
 * amdgpu_lockdep_init() (run at module load) calls fs_reclaim_acquire()
 * while holding the dummy notifier_lock, teaching lockdep that it is legal
 * to enter reclaim with the MMU-notifier lock held. The real notifier lock
 * is taken in amdgpu_hmm_invalidate_gfx(), which mm/ calls from inside
 * reclaim, so the reverse edge fs_reclaim -> mmu_notifier -> notifier_lock
 * is mandatory. The cycle is closed the first time reclaim unmaps a page
 * covered by an amdgpu userptr interval notifier.
 *
 * This program installs such a notifier (GEM_USERPTR) over anonymous memory
 * and then forces synchronous reclaim of that exact range with
 * MADV_PAGEOUT, which runs try_to_unmap()->invalidate_range_start() with
 * fs_reclaim held in the calling thread, closing the loop on demand.
 *
 * Requirements:
 *   - kernel built with CONFIG_PROVE_LOCKING (lockdep)
 *   - amdgpu loaded; run from a FRESH boot (the first lockdep splat of any
 *     kind calls debug_locks_off() and silences all later reports)
 *
 * Build:  cc -O2 -o repro amdgpu-notifier-reclaim-repro.c
 * Run:    ./repro                 # picks the first amdgpu render node
 *         ./repro /dev/dri/renderD129
 * Watch:  sudo dmesg -w
 */

#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <errno.h>
#include <fcntl.h>
#include <unistd.h>
#include <dirent.h>
#include <sys/ioctl.h>
#include <sys/mman.h>

/* --- minimal amdgpu uapi (self-contained, no libdrm needed) ----------- */
#ifndef DRM_IOCTL_BASE
#define DRM_IOCTL_BASE 'd'
#endif
#define DRM_COMMAND_BASE 0x40
#define DRM_AMDGPU_GEM_USERPTR 0x11

struct drm_amdgpu_gem_userptr {
	uint64_t addr;
	uint64_t size;
	uint32_t flags;
	uint32_t handle;
};

#define DRM_IOCTL_AMDGPU_GEM_USERPTR \
	_IOWR(DRM_IOCTL_BASE, DRM_COMMAND_BASE + DRM_AMDGPU_GEM_USERPTR, \
	      struct drm_amdgpu_gem_userptr)

#define AMDGPU_GEM_USERPTR_READONLY (1 << 0)
#define AMDGPU_GEM_USERPTR_ANONONLY (1 << 1)
#define AMDGPU_GEM_USERPTR_VALIDATE (1 << 2)
#define AMDGPU_GEM_USERPTR_REGISTER (1 << 3)

#ifndef MADV_PAGEOUT
#define MADV_PAGEOUT 21
#endif

/* ------------------------------------------------------------ */

#define BUF_SIZE (64ull * 1024 * 1024) /* 64 MiB, page aligned by mmap */

static int open_amdgpu_render(const char *forced)
{
	if (forced) {
		int fd = open(forced, O_RDWR | O_CLOEXEC);
		if (fd < 0)
			perror(forced);
		return fd;
	}

	/* try renderD128..renderD143 and keep the first that accepts GEM_USERPTR */
	for (int i = 128; i < 144; i++) {
		char path[64];
		snprintf(path, sizeof(path), "/dev/dri/renderD%d", i);
		int fd = open(path, O_RDWR | O_CLOEXEC);
		if (fd < 0)
			continue;

		/* probe: a zero-size userptr returns -EINVAL on amdgpu but
		 * -ENOTTY/-ENODEV on a non-amdgpu driver, which lets us tell
		 * the nodes apart without pulling in libdrm version ioctls. */
		struct drm_amdgpu_gem_userptr probe = { 0 };
		errno = 0;
		ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &probe);
		if (errno == ENOTTY || errno == ENODEV || errno == EOPNOTSUPP) {
			close(fd);
			continue;
		}
		fprintf(stderr, "using %s\n", path);
		return fd;
	}
	fprintf(stderr, "no amdgpu render node found under /dev/dri\n");
	return -1;
}

int main(int argc, char **argv)
{
	int fd = open_amdgpu_render(argc > 1 ? argv[1] : NULL);
	if (fd < 0)
		return 1;

	/* anonymous, private, page-aligned region for the userptr */
	void *buf = mmap(NULL, BUF_SIZE, PROT_READ | PROT_WRITE,
			 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
	if (buf == MAP_FAILED) {
		perror("mmap");
		return 1;
	}
	memset(buf, 0xa5, BUF_SIZE); /* fault every page in */

	/* REGISTER installs the mmu_interval_notifier (amdgpu_hmm_register());
	 * VALIDATE additionally faults the pages via hmm_range_fault() and
	 * binds them into GTT. ANONONLY matches our MAP_ANONYMOUS region. */
	struct drm_amdgpu_gem_userptr up = {
		.addr  = (uint64_t)(uintptr_t)buf,
		.size  = BUF_SIZE,
		.flags = AMDGPU_GEM_USERPTR_ANONONLY |
			 AMDGPU_GEM_USERPTR_REGISTER |
			 AMDGPU_GEM_USERPTR_VALIDATE,
	};
	if (ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &up)) {
		perror("GEM_USERPTR (VALIDATE)");
		/* retry without VALIDATE: the notifier is still registered, the
		 * pages are present from the memset, MADV_PAGEOUT still works */
		up.flags = AMDGPU_GEM_USERPTR_ANONONLY |
			   AMDGPU_GEM_USERPTR_REGISTER;
		if (ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &up)) {
			perror("GEM_USERPTR (REGISTER)");
			return 1;
		}
	}
	fprintf(stderr, "userptr handle=%u, interval notifier armed over %p..%p\n",
		up.handle, buf, (char *)buf + BUF_SIZE);

	/* Force synchronous reclaim of the notifier-covered range. MADV_PAGEOUT
	 * runs shrink_folio_list()->try_to_unmap()->invalidate_range_start()
	 * with fs_reclaim held in THIS thread, so amdgpu_hmm_invalidate_gfx()
	 * takes notifier_lock under fs_reclaim and lockdep closes the cycle.
	 *
	 * A few iterations (re-touching in between) cover the race where pages
	 * are already paged out on the first pass. */
	for (int it = 0; it < 8; it++) {
		if (madvise(buf, BUF_SIZE, MADV_PAGEOUT))
			perror("madvise(MADV_PAGEOUT)");
		usleep(50 * 1000);
		memset(buf, 0xa5, BUF_SIZE); /* fault back in for the next pass */
	}

	fprintf(stderr,
		"done: check dmesg for "
		"\"possible circular locking dependency\" / amdgpu_hmm_invalidate_gfx\n");

	munmap(buf, BUF_SIZE);
	close(fd);
	return 0;
}