fsnotify_mark_srcu wtf?

[Miklos Szeredi]

We've got a report where a fanotify daemon that implements permission checks
screws up and doesn't send a reply.  This then causes widespread hangs due to
fsnotify_mark_srcu read side lock being held and thus causing synchronize_srcu()
called from e.g. inotify_release()-> fsnotify_destroy_group()->
fsnotify_mark_destroy_list() to block.

Below program demonstrates the issue.  It should output a single line:

close(inotify_fd): success

Instead it outputs nothing, which means that close(inotify_fd) got blocked by
the waiting permission event.

Wouldn't making the srcu per-group fix this?  Would that be too expensive?

Thanks,
Miklos
---

#include <err.h>
#include <fcntl.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/fanotify.h>
#include <sys/inotify.h>

int main(void)
{
	int fanotify_fd, inotify_fd;
	int ret, pid1, pid2;
	const char *testfile = "t";

	ret = unlink(testfile);
	if (ret == -1 && errno != ENOENT)
		err(1, "unlink()");

	ret = mknod(testfile, S_IFREG | 0666, 0);
	if (ret == -1)
		err(1, "mknod()");

	fanotify_fd = fanotify_init(FAN_CLASS_PRE_CONTENT, O_RDONLY);
	if (fanotify_fd == -1)
		err(1, "fanotify_init()");

	ret = fanotify_mark(fanotify_fd, FAN_MARK_ADD, FAN_OPEN_PERM, AT_FDCWD, testfile);
	if (ret == -1)
		err(1, "fanotify_mark()");

	pid1 = fork();
	if (pid1 == -1)
		err(1, "fork()");

	if (pid1 == 0) {
		close(fanotify_fd);
		ret = open(testfile, O_RDONLY);
		if (ret == -1)
			err(1, "open()");
		fprintf(stderr, "something went wrong: open succeeded\n");
		exit(1);
	}
	sleep(1);

	pid2 = fork();
	if (pid2 == -1)
		err(1, "fork()");

	if (pid2 == 0) {
		close(fanotify_fd);
		inotify_fd = inotify_init();
		if (inotify_fd == -1)
			err(1, "inotify_init()");
		close(inotify_fd);
		fprintf(stderr, "close(inotify_fd): success\n");
		exit(0);
	}

	sleep(1);
	kill(pid1, SIGKILL);
	kill(pid2, SIGKILL);

	return 0;
}