fsnotify_mark_srcu wtf?

[Miklos Szeredi]

We've got a report where a fanotify daemon that implements permission checks
screws up and doesn't send a reply.  This then causes widespread hangs due to
fsnotify_mark_srcu read side lock being held and thus causing synchronize_srcu()
called from e.g. inotify_release()-> fsnotify_destroy_group()->
fsnotify_mark_destroy_list() to block.

Below program demonstrates the issue.  It should output a single line:

close(inotify_fd): success

Instead it outputs nothing, which means that close(inotify_fd) got blocked by
the waiting permission event.

Wouldn't making the srcu per-group fix this?  Would that be too expensive?

Thanks,
Miklos
---

#include <err.h>
#include <fcntl.h>
#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/fanotify.h>
#include <sys/inotify.h>

int main(void)
{
        int fanotify_fd, inotify_fd;
        int ret, pid1, pid2;
        const char *testfile = "t";

        ret = unlink(testfile);
        if (ret == -1 && errno != ENOENT)
                err(1, "unlink()");

        ret = mknod(testfile, S_IFREG | 0666, 0);
        if (ret == -1)
                err(1, "mknod()");

        fanotify_fd = fanotify_init(FAN_CLASS_PRE_CONTENT, O_RDONLY);
        if (fanotify_fd == -1)
                err(1, "fanotify_init()");

        ret = fanotify_mark(fanotify_fd, FAN_MARK_ADD, FAN_OPEN_PERM, AT_FDCWD, 
testfile);
        if (ret == -1)
                err(1, "fanotify_mark()");

        pid1 = fork();
        if (pid1 == -1)
                err(1, "fork()");

        if (pid1 == 0) {
                close(fanotify_fd);
                ret = open(testfile, O_RDONLY);
                if (ret == -1)
                        err(1, "open()");
                fprintf(stderr, "something went wrong: open succeeded\n");
                exit(1);
        }
        sleep(1);

        pid2 = fork();
        if (pid2 == -1)
                err(1, "fork()");

        if (pid2 == 0) {
                close(fanotify_fd);
                inotify_fd = inotify_init();
                if (inotify_fd == -1)
                        err(1, "inotify_init()");
                close(inotify_fd);
                fprintf(stderr, "close(inotify_fd): success\n");
                exit(0);
        }

        sleep(1);
        kill(pid1, SIGKILL);
        kill(pid2, SIGKILL);

        return 0;
}