Re: [syzbot] [mm?] INFO: task hung in migrate_pages_batch (4)

[Aveline Noir]

Hi,

The September 19 and 21 console logs show outstanding nbd0 READ requests
before the hung-task report in folio_lock() from migrate_pages_batch().
I tested whether delaying NBD replies could produce the reported wait.

I used 551c722f4080 with the September 19 syzbot config after
olddefconfig. The test held NBD replies for 180 seconds while 64 threads
performed buffered reads and another thread repeatedly wrote to
/proc/sys/vm/compact_memory. It held 57 READ requests (8 MiB). At 10,
30, 60, 120 and 170 seconds, the compaction thread's stack included
folio_wait_bit_common() and migrate_pages_batch(). Once the replies
were sent, the pending compaction write returned; the run completed
with 64/64 reads and 16/16 compaction writes. The immediate-reply
control completed without a long wait.

In a separate run, I set sysctl.kernel.hung_task_timeout_secs=60 on the
guest command line (the report config defaults to 140 seconds). Before
the NBD replies were sent, it printed:

  INFO: task compact-burst:5047 blocked in I/O wait for more than 61 seconds.
  folio_wait_bit_common+0x371/0xaa0
  migrate_pages_batch+0x15b6/0x4610

The kernel then panicked as configured by CONFIG_BOOTPARAM_HUNG_TASK_PANIC=1;
this run did not reach the NBD reply.

On a separate kprobe-enabled diagnostic kernel, I saw the same folio
address in the NBD read path, the compaction wait, and folio_end_read().
This trace was not taken with the syzbot config.

The delayed replies are deliberate. This produces the wait path seen in
the report, but it does not explain why the original NBD requests stalled
or show that they caused the original hang. I have not measured the hit
rate over repeated runs. The C program and VM instructions are attached.
Do the pending NBD reads in the original logs look related to the folio
lock wait?

Thanks,
Aveline
// SPDX-License-Identifier: GPL-2.0-only
/*
 * Bounded /init diagnostic for synchronous compaction versus NBD read I/O.
 *
 * A deliberately delayed NBD server keeps buffered block-device cache folios
 * locked. The delay is an artificial control, not an explanation of syzbot's
 * original NBD stall. Put /no-delay in the initramfs for the immediate-reply
 * control. Run only in an isolated VM with an external timeout.
 *
 * Build: gcc -std=gnu11 -O2 -Wall -Wextra -Werror -static -pthread \
 *   -o nbd-burst-compact-init nbd-burst-compact-init.c
 */

#define _GNU_SOURCE
#include <arpa/inet.h>
#include <endian.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
#include <linux/nbd.h>
#include <poll.h>
#include <pthread.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/prctl.h>
#include <sys/reboot.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <time.h>
#include <unistd.h>

#define NS_PER_SEC 1000000000ULL
#define MIB (1024ULL * 1024)
#define PAGE_BYTES 4096
#define NBD_BYTES (32ULL * MIB)
#define READ_OFFSET (8ULL * MIB)
#define READERS 64
#define READ_BYTES (128 * 1024)
#define HOLD_BYTES ((uint64_t)READERS * READ_BYTES)
#define MIN_HELD_BYTES (4ULL * MIB)
#define HOLD_SEC 180
#define WATCHDOG_SEC 320
#define MAX_PENDING 4096
#define MAX_PENDING_BYTES (64ULL * MIB)
#define MAX_REQUEST_BYTES NBD_BYTES
#define MAX_COMPACT_ATTEMPTS 1024

_Static_assert(sizeof(struct nbd_request) == 28, "NBD request wire size");
_Static_assert(sizeof(struct nbd_reply) == 16, "NBD reply wire size");
_Static_assert(READ_OFFSET + HOLD_BYTES <= NBD_BYTES, "read range in device");

struct pending_read {
	struct nbd_request request;
	uint32_t len;
};

static struct pending_read pending[MAX_PENDING];
static unsigned int pending_count;
static unsigned char *anon_before, *anon_after;
static unsigned char read_buffers[READERS][READ_BYTES];
static int nbd_fd = -1, server_fd = -1;
static bool no_delay;
static atomic_bool target_seen, hold_released, release_requested;
static atomic_bool server_done, server_failed, nbd_done, compact_done;
static atomic_bool compact_inflight, stop_compact, shutdown_requested;
static atomic_bool stack_match_seen, hold_released_at_compact_start;
static atomic_int server_end, nbd_result, compact_result, compact_tid;
static atomic_int reader_tid[READERS], reader_result[READERS];
static atomic_bool reader_done[READERS];
static atomic_uint readers_finished, compacts_started, compacts_finished;
static atomic_uint_fast64_t first_target_ns, compact_start_ns;
static atomic_uint_fast64_t held_at_compact_start, bytes_at_compact_start;
static atomic_uint_fast64_t held_requests, held_bytes, replied_held;
static atomic_uint_fast64_t ordinary_reads, overflow_reads, max_compact_ms;

enum server_end_reason {
	SERVER_RUNNING,
	SERVER_DISCONNECT,
	SERVER_LOCAL_SHUTDOWN,
	SERVER_UNEXPECTED_EOF,
	SERVER_PROTOCOL_ERROR,
};

static uint64_t now_ns(void)
{
	struct timespec ts;

	if (clock_gettime(CLOCK_MONOTONIC, &ts)) {
		perror("clock_gettime");
		return 0;
	}
	return (uint64_t)ts.tv_sec * NS_PER_SEC + ts.tv_nsec;
}

static void sleep_ms(long ms)
{
	struct timespec ts = { .tv_sec = ms / 1000,
			       .tv_nsec = ms % 1000 * 1000000L };

	while (nanosleep(&ts, &ts) && errno == EINTR)
		;
}

static void poweroff_guest(void)
{
	puts("guest: requesting power off");
	fflush(stdout);
	if (reboot(RB_POWER_OFF))
		perror("reboot(RB_POWER_OFF)");
	if (reboot(RB_HALT_SYSTEM))
		perror("reboot(RB_HALT_SYSTEM)");
	for (;;)
		pause();
}

static void start_watchdog(void)
{
	pid_t child = fork();

	if (child < 0) {
		perror("fork watchdog");
		poweroff_guest();
	}
	if (!child) {
		sleep_ms(WATCHDOG_SEC * 1000L);
		puts("watchdog: 320-second guest deadline reached");
		poweroff_guest();
	}
}

static bool make_dir(const char *path)
{
	if (!mkdir(path, 0755) || errno == EEXIST)
		return true;
	perror(path);
	return false;
}

static bool mount_one(const char *source, const char *target,
			      const char *type)
{
	if (!mount(source, target, type, 0, NULL) || errno == EBUSY)
		return true;
	fprintf(stderr, "mount %s: %s\n", target, strerror(errno));
	return false;
}

static bool mount_basics(void)
{
	if (!make_dir("/proc") || !make_dir("/sys") || !make_dir("/dev"))
		return false;
	return mount_one("proc", "/proc", "proc") &&
	       mount_one("sysfs", "/sys", "sysfs") &&
	       mount_one("devtmpfs", "/dev", "devtmpfs");
}

static void use_console(void)
{
	int fd = open("/dev/console", O_RDWR | O_CLOEXEC);

	if (fd >= 0) {
		dup2(fd, STDIN_FILENO);
		dup2(fd, STDOUT_FILENO);
		dup2(fd, STDERR_FILENO);
		if (fd > STDERR_FILENO)
			close(fd);
	}
	setvbuf(stdout, NULL, _IONBF, 0);
	setvbuf(stderr, NULL, _IONBF, 0);
}

static bool fault_anon(unsigned char **area, size_t bytes, const char *phase)
{
	size_t i;

	*area = mmap(NULL, bytes, PROT_READ | PROT_WRITE,
		     MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
	if (*area == MAP_FAILED) {
		*area = NULL;
		perror("mmap anonymous pages");
		return false;
	}
	for (i = 0; i < bytes; i += PAGE_BYTES)
		(*area)[i] = (unsigned char)(i / PAGE_BYTES);
	printf("guest: %s faulted %zu MiB anonymous pages\n",
	       phase, bytes / (size_t)MIB);
	return true;
}

static bool send_all(int fd, const void *buf, size_t len)
{
	const unsigned char *p = buf;
	uint64_t deadline = now_ns() + 10 * NS_PER_SEC;
	size_t sent = 0;

	while (sent < len && now_ns() < deadline) {
		ssize_t nr = send(fd, p + sent, len - sent,
				  MSG_NOSIGNAL | MSG_DONTWAIT);

		if (nr > 0) {
			sent += (size_t)nr;
			continue;
		}
		if (nr < 0 && errno == EINTR)
			continue;
		if (nr < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
			struct pollfd pfd = { .fd = fd, .events = POLLOUT };

			if (poll(&pfd, 1, 100) >= 0)
				continue;
			if (errno == EINTR)
				continue;
		}
		return false;
	}
	return sent == len;
}

static bool reply_to(const struct nbd_request *request, uint32_t error,
		     uint32_t len)
{
	static const char zeros[PAGE_BYTES];
	struct nbd_reply reply = {
		.magic = htonl(NBD_REPLY_MAGIC),
		.error = htonl(error),
		.cookie = request->cookie,
	};

	if (!send_all(server_fd, &reply, sizeof(reply)))
		return false;
	if (error)
		return true;
	while (len) {
		size_t chunk = len < sizeof(zeros) ? len : sizeof(zeros);

		if (!send_all(server_fd, zeros, chunk))
			return false;
		len -= (uint32_t)chunk;
	}
	return true;
}

static bool reply_pending(void)
{
	unsigned int i;
	uint64_t elapsed = now_ns() - atomic_load(&first_target_ns);

	printf("server: releasing %u pending READs after %" PRIu64
	       " ms held_bytes=%" PRIuFAST64 "\n", pending_count,
	       elapsed / 1000000, atomic_load(&held_bytes));
	for (i = 0; i < pending_count; i++) {
		if (!reply_to(&pending[i].request, 0, pending[i].len)) {
			printf("server: reply failed at pending index=%u\n", i);
			return false;
		}
		atomic_fetch_add(&replied_held, 1);
	}
	pending_count = 0;
	atomic_store(&hold_released, true);
	return true;
}

static bool target_range(uint64_t from, uint32_t len)
{
	return from < READ_OFFSET + HOLD_BYTES && from + len > READ_OFFSET;
}

static void *nbd_server(void *arg)
{
	unsigned char header[sizeof(struct nbd_request)];
	size_t have = 0;
	struct nbd_request req;
	uint64_t from, start;
	uint32_t command, len;
	enum server_end_reason reason = SERVER_PROTOCOL_ERROR;
	bool ok = true;

	(void)arg;
	prctl(PR_SET_NAME, "nbd-burst-srv", 0, 0, 0);
	for (;;) {
		struct pollfd pfd = { .fd = server_fd, .events = POLLIN };
		int rc;

		start = atomic_load(&first_target_ns);
		if (!no_delay && pending_count &&
		    (atomic_load(&release_requested) ||
		     now_ns() - start >= HOLD_SEC * NS_PER_SEC)) {
			ok = reply_pending();
			if (!ok)
				break;
		}
		rc = poll(&pfd, 1, 50);
		if (rc < 0 && errno == EINTR)
			continue;
		if (rc < 0) {
			perror("server poll");
			ok = false;
			break;
		}
		if (!rc)
			continue;
		if (pfd.revents & (POLLERR | POLLNVAL)) {
			printf("server: poll revents=0x%x\n", pfd.revents);
			ok = false;
			break;
		}
		if (!(pfd.revents & (POLLIN | POLLHUP)))
			continue;
		while (have < sizeof(header)) {
			ssize_t nr = recv(server_fd, header + have,
					  sizeof(header) - have, MSG_DONTWAIT);

			if (nr > 0) {
				have += (size_t)nr;
				continue;
			}
			if (nr < 0 && errno == EINTR)
				continue;
			if (nr < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
				break;
			if (!nr) {
				reason = atomic_load(&shutdown_requested) ?
					 SERVER_LOCAL_SHUTDOWN : SERVER_UNEXPECTED_EOF;
				printf("server: request stream ended (%d)\n", reason);
				goto done;
			}
			perror("server recv");
			ok = false;
			goto done;
		}
		if (have < sizeof(header))
			continue;
		memcpy(&req, header, sizeof(req));
		have = 0;
		if (ntohl(req.magic) != NBD_REQUEST_MAGIC) {
			puts("server: invalid NBD request magic");
			ok = false;
			break;
		}
		command = ntohl(req.type) & 0xffffU;
		len = ntohl(req.len);
		from = be64toh(req.from);
		if (command == NBD_CMD_DISC) {
			puts("server: disconnect request received");
			reason = SERVER_DISCONNECT;
			break;
		}
		if (command != NBD_CMD_READ) {
			printf("server: unexpected NBD command=%u\n", command);
			ok = false;
			break;
		}
		if (len > MAX_REQUEST_BYTES || from > NBD_BYTES ||
		    len > NBD_BYTES - from) {
			printf("server: invalid read offset=%" PRIu64
			       " len=%u\n", from, len);
			ok = reply_to(&req, EINVAL, 0);
			if (!ok)
				break;
			continue;
		}
		if (target_range(from, len) && !atomic_load(&target_seen)) {
			atomic_store(&first_target_ns, now_ns());
			atomic_store(&target_seen, true);
			printf("server: first target READ offset=%" PRIu64
			       " len=%u mode=%s\n", from, len,
			       no_delay ? "no-delay" : "180s-delay");
		}
		if (!no_delay && target_range(from, len) &&
		    !atomic_load(&hold_released) &&
		    !atomic_load(&release_requested) &&
		    pending_count < MAX_PENDING &&
		    atomic_load(&held_bytes) + len <= MAX_PENDING_BYTES) {
			pending[pending_count].request = req;
			pending[pending_count].len = len;
			pending_count++;
			atomic_fetch_add(&held_requests, 1);
			atomic_fetch_add(&held_bytes, len);
			continue;
		}
		if (!no_delay && target_range(from, len) &&
		    !atomic_load(&hold_released) &&
		    !atomic_load(&release_requested) &&
		    (pending_count == MAX_PENDING ||
		     atomic_load(&held_bytes) + len > MAX_PENDING_BYTES))
			atomic_fetch_add(&overflow_reads, 1);
		if (!target_range(from, len))
			atomic_fetch_add(&ordinary_reads, 1);
		ok = reply_to(&req, 0, len);
		if (!ok)
			break;
	}
done:
	if (pending_count && !reply_pending())
		ok = false;
	if (!ok || reason == SERVER_UNEXPECTED_EOF ||
	    reason == SERVER_PROTOCOL_ERROR)
		atomic_store(&server_failed, true);
	atomic_store(&server_end, reason);
	shutdown(server_fd, SHUT_RDWR);
	atomic_store(&server_done, true);
	printf("server: end=%d failed=%d held=%" PRIuFAST64
	       " replied=%" PRIuFAST64 " ordinary=%" PRIuFAST64
	       " overflow=%" PRIuFAST64 "\n", reason,
	       atomic_load(&server_failed), atomic_load(&held_requests),
	       atomic_load(&replied_held), atomic_load(&ordinary_reads),
	       atomic_load(&overflow_reads));
	return NULL;
}

static void *nbd_do_it(void *arg)
{
	int rc, error;

	(void)arg;
	prctl(PR_SET_NAME, "nbd-burst-it", 0, 0, 0);
	rc = ioctl(nbd_fd, NBD_DO_IT);
	error = errno;
	printf("NBD_DO_IT: result=%d errno=%d\n", rc, rc < 0 ? error : 0);
	atomic_store(&nbd_result, rc < 0 ? -error : 0);
	atomic_store(&nbd_done, true);
	return NULL;
}

static void *read_worker(void *arg)
{
	unsigned int index = (unsigned int)(uintptr_t)arg;
	char name[16];
	uint64_t start = now_ns();
	off_t offset = READ_OFFSET + (off_t)index * READ_BYTES;
	ssize_t nr;
	int fd, error;

	snprintf(name, sizeof(name), "nbd-read-%02u", index);
	prctl(PR_SET_NAME, name, 0, 0, 0);
	atomic_store(&reader_tid[index], (int)syscall(SYS_gettid));
	fd = open("/dev/nbd0", O_RDONLY | O_CLOEXEC);
	if (fd < 0) {
		error = errno;
		printf("reader %u open failed: %s\n", index, strerror(error));
		atomic_store(&reader_result[index], -error);
	} else {
		nr = pread(fd, read_buffers[index], READ_BYTES, offset);
		error = errno;
		printf("reader %u pread offset=%jd result=%zd errno=%d "
		       "duration_ms=%" PRIu64 "\n", index, (intmax_t)offset,
		       nr, nr < 0 ? error : 0,
		       (now_ns() - start) / 1000000);
		atomic_store(&reader_result[index], nr == READ_BYTES ? 0 :
			     (nr < 0 ? -error : -EIO));
		close(fd);
	}
	atomic_store(&reader_done[index], true);
	atomic_fetch_add(&readers_finished, 1);
	return NULL;
}

static void *compact_worker(void *arg)
{
	unsigned int limit = no_delay ? 32 : MAX_COMPACT_ATTEMPTS;
	unsigned int i;
	int final = 0;

	(void)arg;
	prctl(PR_SET_NAME, "compact-burst", 0, 0, 0);
	atomic_store(&compact_tid, (int)syscall(SYS_gettid));
	for (i = 0; i < limit && !atomic_load(&stop_compact); i++) {
		uint64_t start, elapsed;
		ssize_t nr;
		int fd, error;

		fd = open("/proc/sys/vm/compact_memory", O_WRONLY | O_CLOEXEC);
		if (fd < 0) {
			final = -errno;
			printf("compact open failed: %s\n", strerror(errno));
			break;
		}
		start = now_ns();
		atomic_store(&compact_start_ns, start);
		atomic_store(&compact_inflight, true);
		atomic_fetch_add(&compacts_started, 1);
		nr = write(fd, "1\n", 2);
		error = errno;
		elapsed = (now_ns() - start) / 1000000;
		if (elapsed > atomic_load(&max_compact_ms))
			atomic_store(&max_compact_ms, elapsed);
		atomic_fetch_add(&compacts_finished, 1);
		atomic_store(&compact_inflight, false);
		if (nr != 2) {
			final = nr < 0 ? -error : -EIO;
			printf("compact attempt %u failed result=%zd errno=%d\n",
			       i + 1, nr, nr < 0 ? error : 0);
			close(fd);
			break;
		}
		if (elapsed >= 1000 || i < 4 || (i + 1) % 50 == 0)
			printf("compact attempt=%u duration_ms=%" PRIu64 "\n",
			       i + 1, elapsed);
		close(fd);
		sleep_ms(100);
	}
	atomic_store(&compact_result, final);
	atomic_store(&compact_done, true);
	printf("compact: finished started=%u completed=%u max_ms=%" PRIuFAST64
	       " result=%d\n", atomic_load(&compacts_started),
	       atomic_load(&compacts_finished), atomic_load(&max_compact_ms),
	       final);
	return NULL;
}

static bool wait_for_capacity(void)
{
	uint64_t deadline = now_ns() + 15 * NS_PER_SEC;
	unsigned long long sectors;
	FILE *fp;

	while (now_ns() < deadline && !atomic_load(&nbd_done)) {
		fp = fopen("/sys/class/block/nbd0/size", "r");
		if (fp) {
			if (fscanf(fp, "%llu", &sectors) == 1 &&
			    sectors == NBD_BYTES / 512) {
				fclose(fp);
				return true;
			}
			fclose(fp);
		}
		sleep_ms(100);
	}
	return false;
}

static bool configure_nbd(int client_fd)
{
	if (ioctl(nbd_fd, NBD_SET_SOCK, client_fd) ||
	    ioctl(nbd_fd, NBD_SET_BLKSIZE, PAGE_BYTES) ||
	    ioctl(nbd_fd, NBD_SET_SIZE, (unsigned long)NBD_BYTES) ||
	    ioctl(nbd_fd, NBD_SET_FLAGS,
		  NBD_FLAG_HAS_FLAGS | NBD_FLAG_READ_ONLY)) {
		perror("NBD setup ioctl");
		return false;
	}
	return true;
}

static void print_task_file(int tid, const char *name)
{
	char path[128], buf[4096];
	ssize_t nr;
	int fd;

	if (tid <= 0)
		return;
	snprintf(path, sizeof(path), "/proc/self/task/%d/%s", tid, name);
	fd = open(path, O_RDONLY | O_CLOEXEC);
	if (fd < 0) {
		printf("task %d %s: unavailable: %s\n", tid, name,
		       strerror(errno));
		return;
	}
	nr = read(fd, buf, sizeof(buf) - 1);
	if (nr < 0)
		printf("task %d %s: read failed: %s\n", tid, name,
		       strerror(errno));
	else {
		buf[nr] = '\0';
		printf("task %d %s:\n%s\n", tid, name, buf);
		if (!strcmp(name, "stack") &&
		    tid == atomic_load(&compact_tid) &&
		    strstr(buf, "folio_wait_bit_common") &&
		    strstr(buf, "migrate_pages_batch")) {
			atomic_store(&stack_match_seen, true);
			printf("sample: compactor stack_match=seen tid=%d\n", tid);
		}
	}
	close(fd);
}

static void print_vmstat(void)
{
	FILE *fp = fopen("/proc/vmstat", "r");
	char line[160], key[96];
	unsigned long long value;

	if (!fp) {
		perror("/proc/vmstat");
		return;
	}
	while (fgets(line, sizeof(line), fp)) {
		if (sscanf(line, "%95s %llu", key, &value) != 2)
			continue;
		if (!strcmp(key, "compact_migrate_scanned") ||
		    !strcmp(key, "compact_free_scanned") ||
		    !strcmp(key, "pgmigrate_success") ||
		    !strcmp(key, "pgmigrate_fail") ||
		    !strcmp(key, "nr_file_pages") ||
		    !strcmp(key, "nr_active_file") ||
		    !strcmp(key, "nr_inactive_file"))
			printf("vmstat %s=%llu\n", key, value);
	}
	fclose(fp);
}

static void sample(unsigned int elapsed)
{
	unsigned int i;
	uint64_t inflight_ms = atomic_load(&compact_inflight) ?
		(now_ns() - atomic_load(&compact_start_ns)) / 1000000 : 0;

	printf("sample t=%us readers_done=%u/%u held_requests=%" PRIuFAST64
	       " held_bytes=%" PRIuFAST64 " replied=%" PRIuFAST64
	       " compact_started=%u compact_finished=%u compact_inflight_ms=%"
	       PRIu64 " release=%d\n", elapsed,
	       atomic_load(&readers_finished), READERS,
	       atomic_load(&held_requests), atomic_load(&held_bytes),
	       atomic_load(&replied_held), atomic_load(&compacts_started),
	       atomic_load(&compacts_finished), inflight_ms,
	       atomic_load(&hold_released));
	for (i = 0; i < 4; i++) {
		print_task_file(atomic_load(&reader_tid[i]), "wchan");
		print_task_file(atomic_load(&reader_tid[i]), "stack");
	}
	print_task_file(atomic_load(&compact_tid), "wchan");
	print_task_file(atomic_load(&compact_tid), "stack");
	print_vmstat();
}

static const char *end_name(int reason)
{
	switch (reason) {
	case SERVER_RUNNING: return "running";
	case SERVER_DISCONNECT: return "NBD_CMD_DISC";
	case SERVER_LOCAL_SHUTDOWN: return "local shutdown";
	case SERVER_UNEXPECTED_EOF: return "unexpected EOF";
	case SERVER_PROTOCOL_ERROR: return "protocol or transport error";
	default: return "unknown";
	}
}

int main(void)
{
	static const unsigned int checkpoints[] = { 10, 30, 60, 120, 170 };
	pthread_t server_thread, nbd_thread, compact_thread, reader_threads[READERS];
	uint64_t deadline, start;
	unsigned int readers_started = 0, next_checkpoint = 0, i;
	int sv[2] = { -1, -1 }, error;
	bool server_started = false, nbd_started = false;
	bool compactor_started = false, disconnect_sent = false;
	bool success = false;
	const char *stage = "mount filesystems";

	setvbuf(stdout, NULL, _IONBF, 0);
	start_watchdog();
	if (!mount_basics()) {
		puts("guest: startup failed at mount filesystems");
		poweroff_guest();
	}
	use_console();
	no_delay = access("/no-delay", F_OK) == 0;
	printf("guest: NBD multi-page compaction diagnostic mode=%s "
	       "readers=%u read_kib=%u hold_s=%u\n",
	       no_delay ? "no-delay" : "180s-delay", READERS,
	       READ_BYTES / 1024, HOLD_SEC);
	stage = "fault initial anonymous pages";
	if (!fault_anon(&anon_before, 128 * MIB, "before NBD"))
		goto out;
	stage = "socketpair";
	if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv)) {
		perror("socketpair");
		goto out;
	}
	stage = "open /dev/nbd0";
	nbd_fd = open("/dev/nbd0", O_RDWR | O_CLOEXEC);
	if (nbd_fd < 0) {
		perror("open /dev/nbd0");
		goto out;
	}
	stage = "NBD setup ioctls";
	if (!configure_nbd(sv[0]))
		goto out;
	close(sv[0]);
	sv[0] = -1;
	server_fd = sv[1];
	sv[1] = -1;
	stage = "start NBD server";
	error = pthread_create(&server_thread, NULL, nbd_server, NULL);
	if (error) {
		printf("pthread_create server: %s\n", strerror(error));
		goto out;
	}
	server_started = true;
	stage = "start NBD_DO_IT";
	error = pthread_create(&nbd_thread, NULL, nbd_do_it, NULL);
	if (error) {
		printf("pthread_create NBD_DO_IT: %s\n", strerror(error));
		goto out;
	}
	nbd_started = true;
	stage = "wait for NBD capacity";
	if (!wait_for_capacity()) {
		puts("guest: NBD capacity did not appear within 15 seconds");
		goto out;
	}
	puts("guest: /dev/nbd0 capacity ready");
	stage = "start buffered readers";
	for (i = 0; i < READERS; i++) {
		error = pthread_create(&reader_threads[i], NULL, read_worker,
				       (void *)(uintptr_t)i);
		if (error) {
			printf("pthread_create reader %u: %s\n", i,
			       strerror(error));
			goto out;
		}
		readers_started++;
	}
	stage = "wait for target READ";
	deadline = now_ns() + 15 * NS_PER_SEC;
	while (!atomic_load(&target_seen) && !atomic_load(&server_failed) &&
	       now_ns() < deadline)
		sleep_ms(50);
	if (!atomic_load(&target_seen)) {
		puts("guest: no target NBD READ observed");
		goto out;
	}
	start = atomic_load(&first_target_ns);
	if (!no_delay) {
		deadline = now_ns() + 5 * NS_PER_SEC;
		while (atomic_load(&held_bytes) < MIN_HELD_BYTES &&
		       !atomic_load(&server_failed) && now_ns() < deadline)
			sleep_ms(50);
		printf("guest: queued before compaction requests=%" PRIuFAST64
		       " bytes=%" PRIuFAST64 "\n",
		       atomic_load(&held_requests), atomic_load(&held_bytes));
	}
	stage = "fault later anonymous pages";
	if (!fault_anon(&anon_after, 256 * MIB, "after NBD"))
		goto out;
	stage = "start synchronous compaction";
	atomic_store(&held_at_compact_start, atomic_load(&held_requests));
	atomic_store(&bytes_at_compact_start, atomic_load(&held_bytes));
	atomic_store(&hold_released_at_compact_start,
		     atomic_load(&hold_released));
	printf("guest: compactor start held_requests=%" PRIuFAST64
	       " held_bytes=%" PRIuFAST64 " hold_released=%d\n",
	       atomic_load(&held_at_compact_start),
	       atomic_load(&bytes_at_compact_start),
	       atomic_load(&hold_released_at_compact_start));
	error = pthread_create(&compact_thread, NULL, compact_worker, NULL);
	if (error) {
		printf("pthread_create compactor: %s\n", strerror(error));
		goto out;
	}
	compactor_started = true;
	stage = "observe pending reads and compaction";
	deadline = no_delay ? now_ns() + 60 * NS_PER_SEC :
		start + (HOLD_SEC + 25) * NS_PER_SEC;
	while (now_ns() < deadline) {
		unsigned int elapsed = (unsigned int)((now_ns() - start) /
							NS_PER_SEC);

		if (next_checkpoint < sizeof(checkpoints) / sizeof(checkpoints[0]) &&
		    elapsed >= checkpoints[next_checkpoint]) {
			sample(elapsed);
			next_checkpoint++;
		}
		if (atomic_load(&server_failed))
			break;
		if (no_delay) {
			if (atomic_load(&readers_finished) == readers_started &&
			    atomic_load(&compact_done))
				break;
		} else if (atomic_load(&hold_released)) {
			atomic_store(&stop_compact, true);
			if (atomic_load(&readers_finished) == readers_started &&
			    atomic_load(&compact_done))
				break;
		}
		sleep_ms(100);
	}
	sample((unsigned int)((now_ns() - start) / NS_PER_SEC));
	out:
	atomic_store(&release_requested, true);
	atomic_store(&stop_compact, true);
	deadline = now_ns() + 10 * NS_PER_SEC;
	while (server_started && atomic_load(&held_requests) &&
	       !atomic_load(&hold_released) && now_ns() < deadline)
		sleep_ms(100);
	deadline = now_ns() + 20 * NS_PER_SEC;
	while ((atomic_load(&readers_finished) < readers_started ||
		(compactor_started && !atomic_load(&compact_done))) &&
	       now_ns() < deadline)
		sleep_ms(100);
	if (nbd_fd >= 0 && nbd_started &&
	    atomic_load(&readers_finished) == readers_started &&
	    (!compactor_started || atomic_load(&compact_done))) {
		if (ioctl(nbd_fd, NBD_DISCONNECT))
			perror("NBD_DISCONNECT");
		else
			disconnect_sent = true;
		deadline = now_ns() + 5 * NS_PER_SEC;
		while ((!atomic_load(&nbd_done) || !atomic_load(&server_done)) &&
		       now_ns() < deadline)
			sleep_ms(100);
		if (atomic_load(&nbd_done) &&
		    ioctl(nbd_fd, NBD_CLEAR_SOCK))
			perror("NBD_CLEAR_SOCK");
	}
	if (server_fd >= 0 && !atomic_load(&server_done)) {
		atomic_store(&shutdown_requested, true);
		shutdown(server_fd, SHUT_RDWR);
	}
	if (server_fd >= 0)
		close(server_fd);
	if (sv[0] >= 0)
		close(sv[0]);
	if (sv[1] >= 0)
		close(sv[1]);
	if (nbd_fd >= 0)
		close(nbd_fd);
	for (i = 0; i < readers_started; i++) {
		if (atomic_load(&reader_done[i]))
			pthread_join(reader_threads[i], NULL);
	}
	if (compactor_started && atomic_load(&compact_done))
		pthread_join(compact_thread, NULL);
	if (nbd_started && atomic_load(&nbd_done))
		pthread_join(nbd_thread, NULL);
	if (server_started && atomic_load(&server_done))
		pthread_join(server_thread, NULL);
	success = atomic_load(&target_seen) &&
		  (no_delay || (atomic_load(&held_requests) ==
				atomic_load(&replied_held) &&
				atomic_load(&hold_released))) &&
		  atomic_load(&readers_finished) == READERS &&
		  atomic_load(&compact_done) && !atomic_load(&compact_result) &&
		  !atomic_load(&server_failed) &&
		  disconnect_sent && atomic_load(&nbd_done) &&
		  !atomic_load(&nbd_result) && atomic_load(&server_done) &&
		  atomic_load(&server_end) == SERVER_DISCONNECT;
	for (i = 0; i < READERS; i++) {
		if (!atomic_load(&reader_done[i]) ||
		    atomic_load(&reader_result[i]))
			success = false;
	}
	printf("guest: final mode=%s stage=%s completed=%d "
	       "readers=%u/%u held=%" PRIuFAST64 " replied=%" PRIuFAST64
	       " bytes=%" PRIuFAST64 " overflow=%" PRIuFAST64
	       " held_at_compact_start=%" PRIuFAST64
	       " bytes_at_compact_start=%" PRIuFAST64
	       " coverage_bytes_met=%s hold_released_at_compact_start=%d"
	       " stack_match=%s"
	       " compacts=%u/%u max_compact_ms=%" PRIuFAST64
	       " compaction_result=%d server=%s server_failed=%d "
	       "nbd_done=%d nbd_result=%d\n",
	       no_delay ? "no-delay" : "180s-delay", stage, success,
	       atomic_load(&readers_finished), READERS,
	       atomic_load(&held_requests), atomic_load(&replied_held),
	       atomic_load(&held_bytes), atomic_load(&overflow_reads),
	       atomic_load(&held_at_compact_start),
	       atomic_load(&bytes_at_compact_start),
	       no_delay ? "n/a" :
	       (atomic_load(&bytes_at_compact_start) >= MIN_HELD_BYTES ?
		"yes" : "no"),
	       atomic_load(&hold_released_at_compact_start),
	       atomic_load(&stack_match_seen) ? "seen" : "unverified",
	       atomic_load(&compacts_finished), atomic_load(&compacts_started),
	       atomic_load(&max_compact_ms), atomic_load(&compact_result),
	       end_name(atomic_load(&server_end)), atomic_load(&server_failed),
	       atomic_load(&nbd_done), atomic_load(&nbd_result));
	poweroff_guest();
	return 0;
}
NBD read/compaction diagnostic for syzbot f523acb98cd782b4872e
===============================================================

This program intentionally delays replies from a guest-local NBD server for
180 seconds while 64 buffered readers access /dev/nbd0 and another thread
writes to /proc/sys/vm/compact_memory. It demonstrates a long wait at
folio_lock() in migrate_pages_batch(). It does not explain why the NBD reads
in the original syzbot crashes stalled, and its hit rate has not been measured
over repeated runs.

Source: nbd-burst-compact-init.c
SHA256: 383d00899cf0091cd58f166164d26b91aa4e759bbd5de1597f29dacc006ddb14

Local test kernel: 551c722f40809618230001baccf219193e22fc5a
Original syzbot config used as input to olddefconfig:
https://syzkaller.appspot.com/text?tag=KernelConfig&x=84649d3ff8d550cb
Download that text as syzbot.config before running the commands below.
The locally resolved .config has SHA256
9e7df6a53bc576191a62d8d919fdbac75742e92add98de8f80dfb11e796a9765.
It preserves CONFIG_BLK_DEV_NBD=y, CONFIG_COMPACTION=y, CONFIG_PREEMPT=y,
CONFIG_KASAN=y, and CONFIG_BOOTPARAM_HUNG_TASK_PANIC=1.

Use a separate checkout at the tested kernel commit. Build with an
out-of-tree output directory and an absolute O= path:

    mkdir -p /absolute/path/to/kernel-build
    cp syzbot.config /absolute/path/to/kernel-build/.config
    make -C /absolute/path/to/linux O=/absolute/path/to/kernel-build olddefconfig
    make -C /absolute/path/to/linux O=/absolute/path/to/kernel-build -j8 bzImage

From the directory containing the attached C file, build a static /init and
pack an initramfs containing that file:

    mkdir -p /absolute/path/to/guest
    gcc -std=gnu11 -O2 -Wall -Wextra -Werror -static -pthread \
        -o /absolute/path/to/guest/init nbd-burst-compact-init.c
    (cd /absolute/path/to/guest && find . -print0 | sort -z | \
        cpio --null -o -H newc | gzip -9 -n) > /absolute/path/to/nbd-burst-delay.cpio.gz

For the immediate-reply control, make a separate guest directory with the
same /init and add an empty /no-delay file before packing its initramfs.

Run in an isolated KVM guest, with no network or host disk:

    timeout --signal=TERM --kill-after=5s 450s qemu-system-x86_64 \
        -accel kvm -cpu qemu64 -m 2048 \
        -smp 4,sockets=4,cores=1,threads=1 \
        -display none -serial stdio -monitor none -no-reboot -nic none \
        -kernel /absolute/path/to/kernel-build/arch/x86/boot/bzImage \
        -initrd /absolute/path/to/nbd-burst-delay.cpio.gz \
        -append 'console=ttyS0 rdinit=/init panic=1 loglevel=7' \
        > /absolute/path/to/serial.log 2>&1

The four single-core sockets allow the config's built-in numa=fake=2 and
panic_on_warn=1 command line to boot without a topology warning in this QEMU
setup. Inspect the serial log; a QEMU exit status of 0 alone is insufficient.

In one local delayed run, the compaction thread remained in
folio_wait_bit_common -> migrate_pages_batch at 10, 30, 60, 120, and 170
seconds. The pending NBD requests received valid replies after about 180
seconds; 64/64 readers and the compaction write then completed. This run did
not print an INFO hung-task message at the config's 140-second timeout.

In a separate run, adding 'sysctl.kernel.hung_task_timeout_secs=60' to the
guest -append string produced the matching INFO message and call trace. The
kernel then panicked as specified by CONFIG_BOOTPARAM_HUNG_TASK_PANIC=1,
before the 180-second NBD reply. That run did not test recovery.

The no-delay control completed all 64 reads and 32 compaction writes without
a long wait. A separate KPROBE-enabled diagnostic kernel showed one folio
address in the NBD read path, the compaction wait, and folio_end_read(). That
address observation was not made under the syzbot report config.