Add a selftest for the "cgroup-v2/freezer: Print information about unfreezable process" change: a process stuck uninterruptibly in the kernel can not be frozen, and once the freeze exceeds the timeout the kernel must log a warning naming the guilty process and dumping its stack.
The test uses a helper module (cg_freezer_hang) that parks a task in an uninterruptible in-kernel sleep, so the cgroup can never freeze. It shrinks the freeze_cgroup_timeout sysctl to keep the test fast, triggers the warning by reading cgroup.events, and verifies the warning line and the captured stack trace via /dev/kmsg. Note: requires kernel-devel package to build the helper modules. https://virtuozzo.atlassian.net/browse/VSTOR-119676 Feature: cgroup/freeze: enhance logging Signed-off-by: Pavel Tikhomirov <[email protected]> --- tools/testing/selftests/cgroup/config | 1 + tools/testing/selftests/cgroup/test_freezer.c | 207 ++++++++++++++++++ .../selftests/cgroup/test_modules/Makefile | 3 +- .../cgroup/test_modules/cg_freezer_hang.c | 86 ++++++++ 4 files changed, 296 insertions(+), 1 deletion(-) create mode 100644 tools/testing/selftests/cgroup/test_modules/cg_freezer_hang.c diff --git a/tools/testing/selftests/cgroup/config b/tools/testing/selftests/cgroup/config index c87a6df7edb53..619c56ae4bc99 100644 --- a/tools/testing/selftests/cgroup/config +++ b/tools/testing/selftests/cgroup/config @@ -10,3 +10,4 @@ CONFIG_TRANSPARENT_HUGEPAGE=y CONFIG_CACHESTAT_SYSCALL=y CONFIG_MODULES=y CONFIG_MODULE_UNLOAD=y +CONFIG_STACKTRACE=y diff --git a/tools/testing/selftests/cgroup/test_freezer.c b/tools/testing/selftests/cgroup/test_freezer.c index 0adc5b6c4ac78..a28b68950db6e 100644 --- a/tools/testing/selftests/cgroup/test_freezer.c +++ b/tools/testing/selftests/cgroup/test_freezer.c @@ -10,6 +10,8 @@ #include <stdlib.h> #include <string.h> #include <sys/wait.h> +#include <fcntl.h> +#include <signal.h> #include "../kselftest.h" #include "cgroup_util.h" @@ -21,6 +23,8 @@ #define debug(args...) #endif +#define FREEZE_TIMEOUT_SYSCTL "/proc/sys/kernel/freeze_cgroup_timeout" + /* * Directory containing the test binary. The helper modules live in a * test_modules/ subdirectory next to it (rsync'd there by TEST_GEN_MODS_DIR), @@ -76,6 +80,105 @@ static void mod_unload(const char *mod) debug("Failed to unload module %s\n", mod); } +/* + * Read/write the freezer timeout sysctl (the unit is jiffies). Used to + * shrink the default 30s timeout so the unfreezable test runs quickly. + */ +static int read_freeze_timeout(void) +{ + int v = -1; + FILE *f; + + f = fopen(FREEZE_TIMEOUT_SYSCTL, "r"); + if (!f) + return -1; + if (fscanf(f, "%d", &v) != 1) + v = -1; + fclose(f); + return v; +} + +static void write_freeze_timeout(int v) +{ + FILE *f; + + f = fopen(FREEZE_TIMEOUT_SYSCTL, "w"); + if (!f) + return; + fprintf(f, "%d", v); + fclose(f); +} + +/* + * Open /dev/kmsg positioned past the last existing record, so that later + * reads return only messages logged from now on. This avoids matching stale + * warnings from earlier runs (pids get reused) and scanning the whole buffer. + * Returns the fd, or -1 on error. + */ +static int kmsg_open_tail(void) +{ + int fd; + + fd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); + if (fd < 0) + return -1; + lseek(fd, 0, SEEK_END); + return fd; +} + +/* + * Verify the freeze-timeout warning for @pid in the kernel log read from + * @kfd (see kmsg_open_tail()): the kernel must name @pid as the unfreezable + * process and then dump its stack, which has to point back into the + * cg_freezer_hang helper module. Returns 0 if both are found. + */ +static int dmesg_check_unfreezable(int kfd, int pid) +{ + int found_warn = 0, found_stack = 0; + char rec[8192]; + char needle[128]; + ssize_t n; + + snprintf(needle, sizeof(needle), + "due to unfreezable process %d:", pid); + + /* Each read() returns one record: "<prefix>;<message>\n". */ + while ((n = read(kfd, rec, sizeof(rec) - 1)) > 0) { + char *msg; + + rec[n] = '\0'; + msg = strchr(rec, ';'); + msg = msg ? msg + 1 : rec; + + if (!found_warn) { + if (strstr(msg, needle)) + found_warn = 1; + continue; + } + /* + * The stack trace is printed right after the warning, one + * "[<addr>] symbol" frame per record. Stop at the first + * record that is no longer a stack frame. + */ + if (!strstr(msg, "[<")) + break; + if (strstr(msg, "cg_freezer_hang")) { + found_stack = 1; + break; + } + } + + if (!found_warn) { + debug("No freeze-timeout warning for pid %d in dmesg\n", pid); + return -1; + } + if (!found_stack) { + debug("No cg_freezer_hang frame in stack of pid %d\n", pid); + return -1; + } + return 0; +} + /* * Check if the cgroup is frozen by looking at the cgroup.events::frozen value. */ @@ -859,6 +962,109 @@ static int test_cgfreezer_vfork(const char *root) return ret; } +/* + * Open /proc/cg_freezer_hang and write to it. The write blocks deep in the + * kernel in an uninterruptible sleep and only returns once the helper module + * is unloaded, so this task can never be frozen by the cgroup freezer. + */ +static int unfreezable_fn(const char *cgroup, void *arg) +{ + int fd; + + fd = open("/proc/cg_freezer_hang", O_WRONLY); + if (fd < 0) + return EXIT_FAILURE; + + /* Blocks in the kernel until cg_freezer_hang is unloaded. */ + if (write(fd, "x", 1) != 1) { + close(fd); + return EXIT_FAILURE; + } + + close(fd); + return EXIT_SUCCESS; +} + +/* + * Test that a cgroup holding a process which is stuck uninterruptibly in the + * kernel fails to freeze, and that the kernel logs a freeze-timeout warning + * naming the guilty process. + */ +static int test_cgfreezer_unfreezable(const char *root) +{ + int ret = KSFT_FAIL; + char *cgroup = NULL; + int saved_timeout = -1; + int pid = -1; + int kfd = -1; + + if (mod_load("cg_freezer_hang")) + return KSFT_SKIP; + + cgroup = cg_name(root, "cg_test_unfreezable"); + if (!cgroup) + goto cleanup; + + if (cg_create(cgroup)) + goto cleanup; + + /* + * Shrink the freeze timeout (jiffies) so we don't wait the 30s + * default. A small value is short for any reasonable HZ. + */ + saved_timeout = read_freeze_timeout(); + write_freeze_timeout(50); + + pid = cg_run_nowait(cgroup, unfreezable_fn, NULL); + if (pid < 0) + goto cleanup; + + if (cg_wait_for_proc_count(cgroup, 1)) + goto cleanup; + + /* Let the task actually enter the in-kernel hang. */ + sleep(1); + + /* Start freezing. It can never complete: the task is unfreezable. */ + if (cg_freeze_nowait(cgroup, true)) + goto cleanup; + + /* Wait well past the (shrunk) timeout. */ + sleep(3); + + /* Capture only kernel log messages emitted from here on. */ + kfd = kmsg_open_tail(); + + /* + * Reading cgroup.events runs check_freeze_timeout() in the kernel, + * which emits the warning. The cgroup must still report not frozen. + */ + if (cg_check_frozen(cgroup, false)) + goto cleanup; + + /* The kernel must name the guilty process and dump its stack. */ + if (kfd >= 0 && dmesg_check_unfreezable(kfd, pid) == 0) + ret = KSFT_PASS; + +cleanup: + if (kfd >= 0) + close(kfd); + /* Thaw first, then unload the module to release the parked task. */ + if (cgroup) + cg_freeze_nowait(cgroup, false); + if (saved_timeout >= 0) + write_freeze_timeout(saved_timeout); + mod_unload("cg_freezer_hang"); + if (pid > 0) { + kill(pid, SIGKILL); + waitpid(pid, NULL, 0); + } + if (cgroup) + cg_destroy(cgroup); + free(cgroup); + return ret; +} + /* * Read the pid of the spare kernel thread spawned by cg_freezer_kthread. */ @@ -940,6 +1146,7 @@ struct cgfreezer_test { T(test_cgfreezer_stopped), T(test_cgfreezer_ptraced), T(test_cgfreezer_vfork), + T(test_cgfreezer_unfreezable), T(test_cgfreezer_kthread), }; #undef T diff --git a/tools/testing/selftests/cgroup/test_modules/Makefile b/tools/testing/selftests/cgroup/test_modules/Makefile index dd401a4345d4b..3f39eeda3a92b 100644 --- a/tools/testing/selftests/cgroup/test_modules/Makefile +++ b/tools/testing/selftests/cgroup/test_modules/Makefile @@ -1,7 +1,8 @@ TESTMODS_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) KDIR ?= /lib/modules/$(shell uname -r)/build -obj-m += cg_freezer_kthread.o +obj-m += cg_freezer_hang.o \ + cg_freezer_kthread.o # Ensure that KDIR exists, otherwise skip the compilation modules: diff --git a/tools/testing/selftests/cgroup/test_modules/cg_freezer_hang.c b/tools/testing/selftests/cgroup/test_modules/cg_freezer_hang.c new file mode 100644 index 0000000000000..0f54d7d4cd4be --- /dev/null +++ b/tools/testing/selftests/cgroup/test_modules/cg_freezer_hang.c @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Helper module for the cgroup-v2 freezer selftest + * "test_cgfreezer_unfreezable". + * + * It exposes /proc/cg_freezer_hang. A task that writes to (or reads from) + * this file gets parked deep in the kernel in an uninterruptible sleep and + * never returns to userspace until the module is unloaded. Such a task can + * not be frozen by the cgroup freezer (the freeze is only acted upon on the + * way back to userspace), so a freezer cgroup holding it stays unfrozen and + * eventually trips the freeze-timeout warning. + * + * The sleep is done in short uninterruptible slices rather than one long + * sleep or a busy loop, so that: + * - the CPU is not spun -> no soft/hard lockups; + * - the task voluntarily context-switches every slice, advancing its + * switch count -> the hung-task detector never sees 120s of "no + * progress" and stays quiet; + * - on rmmod we just raise a stop flag and the parked tasks drain within + * one slice, so cleanup is reliable. + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/proc_fs.h> + +/* uninterruptible sleep slice (1 second) */ +#define HANG_SLICE HZ + +static bool stop; + +static void cg_freezer_hang(void) +{ + pr_info("pid %d parked in kernel\n", task_pid_nr(current)); + + while (!READ_ONCE(stop)) + schedule_timeout_uninterruptible(HANG_SLICE); + + pr_info("pid %d released\n", task_pid_nr(current)); +} + +static ssize_t cg_freezer_hang_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + cg_freezer_hang(); + return count; +} + +static ssize_t cg_freezer_hang_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + cg_freezer_hang(); + return 0; +} + +static const struct proc_ops cg_freezer_hang_ops = { + .proc_write = cg_freezer_hang_write, + .proc_read = cg_freezer_hang_read, +}; + +static int __init cg_freezer_hang_init(void) +{ + if (!proc_create("cg_freezer_hang", 0666, NULL, &cg_freezer_hang_ops)) + return -ENOMEM; + return 0; +} + +static void __exit cg_freezer_hang_exit(void) +{ + /* + * Raise 'stop' first, then remove the entry. remove_proc_entry() + * blocks until all in-flight ->proc_write/->proc_read calls return, + * which only happens once 'stop' is set -- so it also drains the + * parked tasks for us. + */ + WRITE_ONCE(stop, true); + remove_proc_entry("cg_freezer_hang", NULL); +} + +module_init(cg_freezer_hang_init); +module_exit(cg_freezer_hang_exit); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Pavel Tikhomirov <[email protected]>"); +MODULE_DESCRIPTION("cgroup freezer test: park a task unfreezably in kernel"); -- 2.54.0 _______________________________________________ Devel mailing list [email protected] https://lists.openvz.org/mailman/listinfo/devel
