Re: [PATCH] drm/amdgpu: do not enter fs_reclaim under notifier_lock in lockdep training

[vitaly prosyak]

Hi Mikhail,

Thank you so much for providing the deterministic reproducer! This is extremely helpful - I really appreciate the detailed build notes and explanation of the lockdep cycle.

Ill test this on my side this week and confirm the splat. Once we have the fix ready and tested, Ill be sure to credit you with Tested-by and acknowledge your help in identifying and reproducing the issue.

Your assistance is much appreciated!

Best regards,
Vitaly

On 2026-06-19 08:18, Mikhail Gavrilov wrote:
> [You don't often get email from mikhail.v.gavrilov@xxxxxxxxx. Learn why this is important at https://aka.ms/LearnAboutSenderIdentification ]
>
> Makes sense, thanks. I won't respin this one then.
>
> Vitaly, for the reorder, here is a deterministic reproducer so you can confirm
> the splat on your side without a round-trip. It arms an mmu_interval_notifier
> via GEM_USERPTR over anonymous memory, then forces reclaim of that exact range
> with madvise(MADV_PAGEOUT), so amdgpu_hmm_invalidate_gfx() takes notifier_lock
> under fs_reclaim in the calling thread. Needs CONFIG_PROVE_LOCKING and a fresh
> boot; build/run notes are in the header. Happy to give Tested-by once you post.
>
> // SPDX-License-Identifier: MIT
> /*
>  * amdgpu-notifier-reclaim-repro.c
>  *
>  * Deterministic reproducer for the false circular-locking-dependency splat
>  * produced by drivers/gpu/drm/amd/amdgpu/amdgpu_lockdep.c.
>  *
>  * amdgpu_lockdep_init() (run at module load) calls fs_reclaim_acquire()
>  * while holding the dummy notifier_lock, teaching lockdep that it is legal
>  * to enter reclaim with the MMU-notifier lock held. The real notifier lock
>  * is taken in amdgpu_hmm_invalidate_gfx(), which mm/ calls from inside
>  * reclaim, so the reverse edge fs_reclaim -> mmu_notifier -> notifier_lock
>  * is mandatory. The cycle is closed the first time reclaim unmaps a page
>  * covered by an amdgpu userptr interval notifier.
>  *
>  * This program installs such a notifier (GEM_USERPTR) over anonymous memory
>  * and then forces synchronous reclaim of that exact range with
>  * MADV_PAGEOUT, which runs try_to_unmap()->invalidate_range_start() with
>  * fs_reclaim held in the calling thread, closing the loop on demand.
>  *
>  * Requirements:
>  *   - kernel built with CONFIG_PROVE_LOCKING (lockdep)
>  *   - amdgpu loaded; run from a FRESH boot (the first lockdep splat of any
>  *     kind calls debug_locks_off() and silences all later reports)
>  *
>  * Build:  cc -O2 -o repro amdgpu-notifier-reclaim-repro.c
>  * Run:    ./repro                 # picks the first amdgpu render node
>  *         ./repro /dev/dri/renderD129
>  * Watch:  sudo dmesg -w
>  */
>
> #define _GNU_SOURCE
> #include <stdio.h>
> #include <stdlib.h>
> #include <string.h>
> #include <stdint.h>
> #include <errno.h>
> #include <fcntl.h>
> #include <unistd.h>
> #include <dirent.h>
> #include <sys/ioctl.h>
> #include <sys/mman.h>
>
> /* --- minimal amdgpu uapi (self-contained, no libdrm needed) ----------- */
> #ifndef DRM_IOCTL_BASE
> #define DRM_IOCTL_BASE 'd'
> #endif
> #define DRM_COMMAND_BASE 0x40
> #define DRM_AMDGPU_GEM_USERPTR 0x11
>
> struct drm_amdgpu_gem_userptr {
>         uint64_t addr;
>         uint64_t size;
>         uint32_t flags;
>         uint32_t handle;
> };
>
> #define DRM_IOCTL_AMDGPU_GEM_USERPTR \
>         _IOWR(DRM_IOCTL_BASE, DRM_COMMAND_BASE + DRM_AMDGPU_GEM_USERPTR, \
>               struct drm_amdgpu_gem_userptr)
>
> #define AMDGPU_GEM_USERPTR_READONLY (1 << 0)
> #define AMDGPU_GEM_USERPTR_ANONONLY (1 << 1)
> #define AMDGPU_GEM_USERPTR_VALIDATE (1 << 2)
> #define AMDGPU_GEM_USERPTR_REGISTER (1 << 3)
>
> #ifndef MADV_PAGEOUT
> #define MADV_PAGEOUT 21
> #endif
>
> /* ------------------------------------------------------------ */
>
> #define BUF_SIZE (64ull * 1024 * 1024) /* 64 MiB, page aligned by mmap */
>
> static int open_amdgpu_render(const char *forced)
> {
>         if (forced) {
>                 int fd = open(forced, O_RDWR | O_CLOEXEC);
>                 if (fd < 0)
>                         perror(forced);
>                 return fd;
>         }
>
>         /* try renderD128..renderD143 and keep the first that accepts GEM_USERPTR */
>         for (int i = 128; i < 144; i++) {
>                 char path[64];
>                 snprintf(path, sizeof(path), "/dev/dri/renderD%d", i);
>                 int fd = open(path, O_RDWR | O_CLOEXEC);
>                 if (fd < 0)
>                         continue;
>
>                 /* probe: a zero-size userptr returns -EINVAL on amdgpu but
>                  * -ENOTTY/-ENODEV on a non-amdgpu driver, which lets us tell
>                  * the nodes apart without pulling in libdrm version ioctls. */
>                 struct drm_amdgpu_gem_userptr probe = { 0 };
>                 errno = 0;
>                 ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &probe);
>                 if (errno == ENOTTY || errno == ENODEV || errno == EOPNOTSUPP) {
>                         close(fd);
>                         continue;
>                 }
>                 fprintf(stderr, "using %s\n", path);
>                 return fd;
>         }
>         fprintf(stderr, "no amdgpu render node found under /dev/dri\n");
>         return -1;
> }
>
> int main(int argc, char **argv)
> {
>         int fd = open_amdgpu_render(argc > 1 ? argv[1] : NULL);
>         if (fd < 0)
>                 return 1;
>
>         /* anonymous, private, page-aligned region for the userptr */
>         void *buf = mmap(NULL, BUF_SIZE, PROT_READ | PROT_WRITE,
>                          MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
>         if (buf == MAP_FAILED) {
>                 perror("mmap");
>                 return 1;
>         }
>         memset(buf, 0xa5, BUF_SIZE); /* fault every page in */
>
>         /* REGISTER installs the mmu_interval_notifier (amdgpu_hmm_register());
>          * VALIDATE additionally faults the pages via hmm_range_fault() and
>          * binds them into GTT. ANONONLY matches our MAP_ANONYMOUS region. */
>         struct drm_amdgpu_gem_userptr up = {
>                 .addr  = (uint64_t)(uintptr_t)buf,
>                 .size  = BUF_SIZE,
>                 .flags = AMDGPU_GEM_USERPTR_ANONONLY |
>                          AMDGPU_GEM_USERPTR_REGISTER |
>                          AMDGPU_GEM_USERPTR_VALIDATE,
>         };
>         if (ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &up)) {
>                 perror("GEM_USERPTR (VALIDATE)");
>                 /* retry without VALIDATE: the notifier is still registered, the
>                  * pages are present from the memset, MADV_PAGEOUT still works */
>                 up.flags = AMDGPU_GEM_USERPTR_ANONONLY |
>                            AMDGPU_GEM_USERPTR_REGISTER;
>                 if (ioctl(fd, DRM_IOCTL_AMDGPU_GEM_USERPTR, &up)) {
>                         perror("GEM_USERPTR (REGISTER)");
>                         return 1;
>                 }
>         }
>         fprintf(stderr, "userptr handle=%u, interval notifier armed over %p..%p\n",
>                 up.handle, buf, (char *)buf + BUF_SIZE);
>
>         /* Force synchronous reclaim of the notifier-covered range. MADV_PAGEOUT
>          * runs shrink_folio_list()->try_to_unmap()->invalidate_range_start()
>          * with fs_reclaim held in THIS thread, so amdgpu_hmm_invalidate_gfx()
>          * takes notifier_lock under fs_reclaim and lockdep closes the cycle.
>          *
>          * A few iterations (re-touching in between) cover the race where pages
>          * are already paged out on the first pass. */
>         for (int it = 0; it < 8; it++) {
>                 if (madvise(buf, BUF_SIZE, MADV_PAGEOUT))
>                         perror("madvise(MADV_PAGEOUT)");
>                 usleep(50 * 1000);
>                 memset(buf, 0xa5, BUF_SIZE); /* fault back in for the next pass */
>         }
>
>         fprintf(stderr,
>                 "done: check dmesg for "
>                 "\"possible circular locking dependency\" / amdgpu_hmm_invalidate_gfx\n");
>
>         munmap(buf, BUF_SIZE);
>         close(fd);
>         return 0;
> }