When nr_open is already set to INT_MAX, nr_open + 1024 overflows and causes setrlimit to fail. This happens on systems where fs.nr_open has been configured to its maximum value.
Add a check to detect this condition. If nr_open is near INT_MAX, skip increasing fs.nr_open and use a fallback value of 1048576 for RLIMIT_NOFILE instead, which is large enough for the test. This fixes the test failure on systems with maxed-out nr_open. Signed-off-by: Jin Li <[email protected]> --- tools/testing/selftests/core/unshare_test.c | 38 +++++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/core/unshare_test.c b/tools/testing/selftests/core/unshare_test.c index ffce75a6c228..dcc61488e4dd 100644 --- a/tools/testing/selftests/core/unshare_test.c +++ b/tools/testing/selftests/core/unshare_test.c @@ -42,18 +42,34 @@ TEST(unshare_EMFILE) ASSERT_EQ(0, getrlimit(RLIMIT_NOFILE, &rlimit)); - /* bump fs.nr_open */ - n2 = sprintf(buf2, "%d\n", nr_open + 1024); - lseek(fd, 0, SEEK_SET); - write(fd, buf2, n2); - - /* bump ulimit -n */ - rlimit.rlim_cur = nr_open + 1024; - rlimit.rlim_max = nr_open + 1024; - EXPECT_EQ(0, setrlimit(RLIMIT_NOFILE, &rlimit)) { + /* + * Only bump fs.nr_open and RLIMIT_NOFILE if nr_open is not already + * at or near INT_MAX. Adding 1024 to INT_MAX would overflow. + */ + if (nr_open < INT_MAX - 1024) { + n2 = sprintf(buf2, "%d\n", nr_open + 1024); lseek(fd, 0, SEEK_SET); - write(fd, buf, n); - exit(EXIT_FAILURE); + write(fd, buf2, n2); + + rlimit.rlim_cur = nr_open + 1024; + rlimit.rlim_max = nr_open + 1024; + EXPECT_EQ(0, setrlimit(RLIMIT_NOFILE, &rlimit)) { + lseek(fd, 0, SEEK_SET); + write(fd, buf, n); + exit(EXIT_FAILURE); + } + } else { + /* + * If nr_open is already at maximum, use a fallback value + * for RLIMIT_NOFILE that is large enough for the test. + */ + rlimit.rlim_cur = 1048576; + rlimit.rlim_max = 1048576; + EXPECT_EQ(0, setrlimit(RLIMIT_NOFILE, &rlimit)) { + lseek(fd, 0, SEEK_SET); + write(fd, buf, n); + exit(EXIT_FAILURE); + } } /* get a descriptor past the old fs.nr_open */ -- 2.53.0
