As CPU hotplug is now being used to enable runtime update to the list of nohz_full and managed_irq CPUs, we should avoid using CPU 0 in the formation of isolated partition as CPU 0 may not be able to be brought offline like in the case of x86-64 architecture. So a number of the test cases in test_cpuset_prs.sh will have to be updated accordingly.
A new test will also be run in offline isn't allowed in CPU 0 to verify that using CPU 0 as part of an isolated partition will fail. The cgroup-v2.rst is also updated to reflect the new capability of using CPU hotplug to enable run time change to the nohz_full and managed_irq CPU lists. Since there is a slight performance overhead to enable runtime changes to nohz_full CPU list, users have to explicitly opt in by adding a "nohz_ful" kernel command line parameter with or without a CPU list. Signed-off-by: Waiman Long <[email protected]> --- Documentation/admin-guide/cgroup-v2.rst | 35 +++++++--- .../selftests/cgroup/test_cpuset_prs.sh | 70 +++++++++++++++++-- 2 files changed, 92 insertions(+), 13 deletions(-) diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index 8ad0b2781317..e97fc031eb86 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -2604,11 +2604,12 @@ Cpuset Interface Files It accepts only the following input values when written to. - ========== ===================================== + ========== =============================================== "member" Non-root member of a partition "root" Partition root - "isolated" Partition root without load balancing - ========== ===================================== + "isolated" Partition root without load balancing and other + OS noises + ========== =============================================== A cpuset partition is a collection of cpuset-enabled cgroups with a partition root at the top of the hierarchy and its descendants @@ -2652,11 +2653,29 @@ Cpuset Interface Files partition or scheduling domain. The set of exclusive CPUs is determined by the value of its "cpuset.cpus.exclusive.effective". - When set to "isolated", the CPUs in that partition will be in - an isolated state without any load balancing from the scheduler - and excluded from the unbound workqueues. Tasks placed in such - a partition with multiple CPUs should be carefully distributed - and bound to each of the individual CPUs for optimal performance. + When set to "isolated", the CPUs in that partition will be in an + isolated state without any load balancing from the scheduler and + excluded from the unbound workqueues as well as other OS noises. + Tasks placed in such a partition with multiple CPUs should be + carefully distributed and bound to each of the individual CPUs + for optimal performance. + + As CPU hotplug, if supported, is used to improve the degree of + CPU isolation close to the "nohz_full" kernel boot parameter. + In some architectures, like x86-64, the boot CPU (typically CPU + 0) cannot be brought offline, so the boot CPU should not be used + for forming isolated partitions. The "nohz_full" kernel boot + parameter needs to be present to enable full dynticks support + and RCU no-callback CPU mode for CPUs in isolated partitions + even if the optional cpu list isn't provided. + + Using CPU hotplug for creating or destroying an isolated + partition can cause latency spike in applications running + in other isolated partitions. A reserved list of CPUs can + optionally be put in the "nohz_full" kernel boot parameter to + alleviate this problem. When these reserved CPUs are used for + isolated partitions, CPU hotplug won't need to be invoked and + so there won't be latency spike in other isolated partitions. A partition root ("root" or "isolated") can be in one of the two possible states - valid or invalid. An invalid partition diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index a56f4153c64d..eebb4122b581 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -67,6 +67,12 @@ then echo Y > /sys/kernel/debug/sched/verbose fi +# Enable dynamic debug message if available +DYN_DEBUG=/proc/dynamic_debug/control +[[ -f $DYN_DEBUG ]] && { + echo "file kernel/cpu.c +p" > $DYN_DEBUG +} + cd $CGROUP2 echo +cpuset > cgroup.subtree_control @@ -84,6 +90,15 @@ echo member > test/cpuset.cpus.partition echo "" > test/cpuset.cpus [[ $RESULT -eq 0 ]] && skip_test "Child cgroups are using cpuset!" +# +# If nohz_full parameter is specified and nohz_full file exists, CPU hotplug +# will be used to modify nohz_full cpumask to include all the isolated CPUs +# in cpuset isolated partitions. +# +NOHZ_FULL=/sys/devices/system/cpu/nohz_full +BOOT_NOHZ_FULL=$(fmt -1 /proc/cmdline | grep "^nohz_full") +[[ "$BOOT_NOHZ_FULL" = nohz_full ]] && CHK_NOHZ_FULL=1 + # # If isolated CPUs have been reserved at boot time (as shown in # cpuset.cpus.isolated), these isolated CPUs should be outside of CPUs 0-8 @@ -318,8 +333,8 @@ TEST_MATRIX=( # Invalid to valid local partition direct transition tests " C1-3:P2 X4:P2 . . . . . . 0 A1:1-3|XA1:1-3|A2:1-3:XA2: A1:P2|A2:P-2 1-3" " C1-3:P2 X4:P2 . . . X3:P2 . . 0 A1:1-2|XA1:1-3|A2:3:XA2:3 A1:P2|A2:P2 1-3" - " C0-3:P2 . . C4-6 C0-4 . . . 0 A1:0-4|B1:5-6 A1:P2|B1:P0" - " C0-3:P2 . . C4-6 C0-4:C0-3 . . . 0 A1:0-3|B1:4-6 A1:P2|B1:P0 0-3" + " C1-3:P2 . . C4-6 C1-4 . . . 0 A1:1-4|B1:5-6 A1:P2|B1:P0" + " C1-3:P2 . . C4-6 C1-4:C1-3 . . . 0 A1:1-3|B1:4-6 A1:P2|B1:P0 1-3" # Local partition invalidation tests " C0-3:X1-3:P2 C1-3:X2-3:P2 C2-3:X3:P2 \ @@ -329,8 +344,8 @@ TEST_MATRIX=( " C0-3:X1-3:P2 C1-3:X2-3:P2 C2-3:X3:P2 \ . . C4:X . . 0 A1:1-3|A2:1-3|A3:2-3|XA2:|XA3: A1:P2|A2:P-2|A3:P-2 1-3" # Local partition CPU change tests - " C0-5:P2 C4-5:P1 . . . C3-5 . . 0 A1:0-2|A2:3-5 A1:P2|A2:P1 0-2" - " C0-5:P2 C4-5:P1 . . C1-5 . . . 0 A1:1-3|A2:4-5 A1:P2|A2:P1 1-3" + " C1-5:P2 C4-5:P1 . . . C3-5 . . 0 A1:1-2|A2:3-5 A1:P2|A2:P1 1-2" + " C1-5:P2 C4-5:P1 . . C2-5 . . . 0 A1:2-3|A2:4-5 A1:P2|A2:P1 2-3" # cpus_allowed/exclusive_cpus update tests " C0-3:X2-3 C1-3:X2-3 C2-3:X2-3 \ @@ -442,6 +457,21 @@ TEST_MATRIX=( " C0-3 . . C4-5 X3-5 . . . 1 A1:0-3|B1:4-5" ) +# +# Test matrix to verify that using CPU 0 in isolated (local or remote) partition +# will fail when offline isn't allowed for CPU 0. +# +CPU0_ISOLCPUS_MATRIX=( + # old-A1 old-A2 old-A3 old-B1 new-A1 new-A2 new-A3 new-B1 fail ECPUs Pstate ISOLCPUS + # ------ ------ ------ ------ ------ ------ ------ ------ ---- ----- ------ -------- + " C0-3 . . C4-5 P2 . . . 0 A1:0-3|B1:4-5 A1:P-2" + " C1-3 . . . P2 . . . 0 A1:1-3 A1:P2" + " C1-3 . . . P2:C0-3 . . . 0 A1:0-3 A1:P-2" + " CX0-3 C0-3 . . . P2 . . 0 A1:0-3|A2:0-3 A2:P-2" + " CX0-3 C0-3:X1-3 . . . P2 . . 0 A1:0|A2:1-3 A2:P2" + " CX0-3 C0-3:X1-3 . . . P2:X0-3 . . 0 A1:0-3|A2:0-3 A2:P-2" +) + # # Cpuset controller remote partition test matrix. # @@ -513,7 +543,7 @@ write_cpu_online() } fi echo $VAL > $CPUFILE - pause 0.05 + pause 0.10 } # @@ -654,6 +684,8 @@ dump_states() [[ -e $PCPUS ]] && echo "$PCPUS: $(cat $PCPUS)" [[ -e $ISCPUS ]] && echo "$ISCPUS: $(cat $ISCPUS)" done + # Dump nohz_full + [[ -f $NOHZ_FULL ]] && echo "nohz_full: $(cat $NOHZ_FULL)" } # @@ -789,6 +821,18 @@ check_isolcpus() EXPECTED_SDOMAIN=$EXPECTED_ISOLCPUS fi + # + # Check if nohz_full match cpuset.cpus.isolated if nohz_boot parameter + # specified with no parameter. + # + [[ -f $NOHZ_FULL && "$BOOT_NOHZ_FULL" = nohz_full ]] && { + NOHZ_FULL_CPUS=$(cat $NOHZ_FULL) + [[ "$ISOLCPUS" != "$NOHZ_FULL_CPUS" ]] && { + echo "nohz_full ($NOHZ_FULL_CPUS) does not match cpuset.cpus.isolated ($ISOLCPUS)" + return 1 + } + } + # # Appending pre-isolated CPUs # Even though CPU #8 isn't used for testing, it can't be pre-isolated @@ -1070,6 +1114,21 @@ run_remote_state_test() echo "All $I tests of $TEST PASSED." } +# +# Testing CPU 0 isolated partition test when offline is disabled +# +run_cpu0_isol_test() +{ + # Skip the test if CPU0 offline is allowed or if nohz_full kernel + # boot parameter is missing. + CPU0_ONLINE=/sys/devices/system/cpu/cpu0/online + [[ -f $CPU0_ONLINE ]] && return + grep -q -w nohz_full /proc/cmdline + [[ $? -ne 0 ]] && return + + run_state_test CPU0_ISOLCPUS_MATRIX +} + # # Testing the new "isolated" partition root type # @@ -1207,6 +1266,7 @@ test_inotify() trap cleanup 0 2 3 6 run_state_test TEST_MATRIX run_remote_state_test REMOTE_TEST_MATRIX +run_cpu0_isol_test test_isolated test_inotify echo "All tests PASSED." -- 2.53.0
