This patch has not been tested since rebasing onto upstream vmtest.sh.
It is probably very broken, but here to show the direction the testing
is going in...
vsock_test uses tcp for the control socket. TCP itself responds to
namespaces. In order to test vsock but not break TCP, vmtest.sh sets up
a bridge with socat (perhaps ip tables would be better, because it can
reduce an out-of-tree tool dependency). Another option is to not use
vsock_test for the NS tests, but it seems more robust to test all of
vsock instead of just (for example) connectibility...
Signed-off-by: Bobby Eshleman <bobbyeshle...@gmail.com>
---
tools/testing/selftests/vsock/vmtest.sh | 555 +++++++++++++++++++++++++++++---
1 file changed, 510 insertions(+), 45 deletions(-)
diff --git a/tools/testing/selftests/vsock/vmtest.sh
b/tools/testing/selftests/vsock/vmtest.sh
index edacebfc1632..8f627f60cc11 100755
--- a/tools/testing/selftests/vsock/vmtest.sh
+++ b/tools/testing/selftests/vsock/vmtest.sh
@@ -7,6 +7,48 @@
# * virtme-ng
# * busybox-static (used by virtme-ng)
# * qemu (used by virtme-ng)
+#
+# Namespace tests require to test the functionality of VSOCK under different
+# namespace configurations. Ideally, we can use vsock_test and friends under
+# the different configurations to ensure that all functionality works
+# regardless of namespace setup. vsock_test also requires TCP for its control
+# plane, which is also impacted by namespacing. For this reason, these tests
+# build a bridge between the namespaces so that the TCP control traffic can
+# flow between namespaces. The bridge setup looks as follows:
+#
+#
+# |
+# +------------------+ |
+# | VM | |
+# | | NS0 | NS1
+# | +------------+ | |
+# | | | | --------+--------------------+
+# | | vsock_test | | | |
+# | | | | <-------+-----------------+ |
+# | +------------+ | | VSOCK_TEST_PORT| |
+# | | | | | VSOCK
+# +------------------+ | | |
+# ^ | | | |
+# CONTROL_PORT| | | | |
+# | | | | |
+# | | | | v
+# | | | +------------+
+# | | TCP | | |
+# | | | | vsock_test |
+# | | | | |
+# | | | +------------+
+# CONTROL_PORT | | | CONTROL_PORT ^ |
+# | | | | |
+# | v | CONTROL_PORT | v
+# +-------+ | +-------+
+# | |veth0 | veth1| |
+# | socat |<-------------+------------- | socat |
+# | | -------------+------------> | |
+# +-------+ | +-------+
+# NS_BRIDGE_PORT | NS_BRIDGE_PORT
+# |
+
+set -u
readonly SCRIPT_DIR="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
readonly KERNEL_CHECKOUT=$(realpath "${SCRIPT_DIR}"/../../../../)
@@ -19,11 +61,11 @@ readonly TEST_HOST_PORT=50000
readonly TEST_HOST_PORT_LISTENER=50001
readonly SSH_GUEST_PORT=22
readonly SSH_HOST_PORT=2222
-readonly VSOCK_CID=1234
+readonly BRIDGE_PORT=5678
+readonly DEFAULT_CID=1234
readonly WAIT_PERIOD=3
readonly WAIT_PERIOD_MAX=60
-readonly WAIT_TOTAL=$(( WAIT_PERIOD * WAIT_PERIOD_MAX ))
-readonly QEMU_PIDFILE=$(mktemp /tmp/qemu_vsock_vmtest_XXXX.pid)
+WAIT_TOTAL=$(( WAIT_PERIOD * WAIT_PERIOD_MAX ))
# virtme-ng offers a netdev for ssh when using "--ssh", but we also need a
# control port forwarded for vsock_test. Because virtme-ng doesn't support
@@ -33,23 +75,48 @@ readonly QEMU_PIDFILE=$(mktemp
/tmp/qemu_vsock_vmtest_XXXX.pid)
# add the kernel cmdline options that virtme-init uses to setup the interface.
readonly
QEMU_TEST_PORT_FWD="hostfwd=tcp::${TEST_HOST_PORT}-:${TEST_GUEST_PORT}"
readonly QEMU_SSH_PORT_FWD="hostfwd=tcp::${SSH_HOST_PORT}-:${SSH_GUEST_PORT}"
-readonly QEMU_OPTS="\
+readonly LOG=$(mktemp /tmp/vsock_vmtest_XXXX.log)
+readonly TEST_NAMES=(vm_server_host_client vm_client_host_server vm_loopback)
+QEMU_OPTS="\
-netdev user,id=n0,${QEMU_TEST_PORT_FWD},${QEMU_SSH_PORT_FWD} \
-device virtio-net-pci,netdev=n0 \
- -device vhost-vsock-pci,guest-cid=${VSOCK_CID} \
- --pidfile ${QEMU_PIDFILE} \
"
readonly KERNEL_CMDLINE="\
virtme.dhcp net.ifnames=0 biosdevname=0 \
virtme.ssh virtme_ssh_channel=tcp virtme_ssh_user=$USER \
"
readonly LOG=$(mktemp /tmp/vsock_vmtest_XXXX.log)
-readonly TEST_NAMES=(vm_server_host_client vm_client_host_server vm_loopback)
+readonly TEST_NAMES=(
+ vm_server_host_client
+ vm_client_host_server
+ vm_loopback
+ host_vsock_ns_mode
+ host_vsock_ns_mode_write_once
+ global_same_cid
+ local_same_cid
+ global_local_same_cid
+ local_global_same_cid
+ global_host_connect_global_vm
+ global_vm_connect_global_host
+ global_vm_connect_mixed_host
+)
+
readonly TEST_DESCS=(
"Run vsock_test in server mode on the VM and in client mode on the
host."
"Run vsock_test in client mode on the VM and in server mode on the
host."
"Run vsock_test using the loopback transport in the VM."
+ "Check /proc/net/vsock_ns_mode strings on the host."
+ "Check /proc/net/vsock_ns_mode is write-once on the host."
+ "Test that CID allocation fails with the same CID, one global NS and
another global NS."
+ "Test that CID allocation succeeds with the same CID, one local NS and
another local NS."
+ "Test that CID allocation succeeds with the same CID, one global NS and
one local NS, global allocates first."
+ "Test that CID allocation succeeds with the same CID, one global NS and
one local NS, local allocates first."
)
+readonly NEEDS_SETUP=(vm_server_host_client vm_client_host_server vm_loopback)
+readonly MODES=("local" "global" "mixed")
+readonly PIDFILE_TEMPLATE="/tmp/qemu_vsock_vmtest_XXXX.pid"
+
+declare -a PIDFILES
VERBOSE=0
@@ -84,21 +151,40 @@ die() {
exit "${KSFT_FAIL}"
}
+cleanup() {
+ terminate_pidfiles ${PIDFILES[@]}
+ del_namespaces
+}
+
vm_ssh() {
ssh -q -o UserKnownHostsFile=/dev/null -p ${SSH_HOST_PORT} localhost
"$@"
return $?
}
-cleanup() {
- if [[ -s "${QEMU_PIDFILE}" ]]; then
- pkill -SIGTERM -F "${QEMU_PIDFILE}" > /dev/null 2>&1
- fi
+vm_ssh_ns() {
+ local ns="${1}"
+ local NS_EXEC="ip netns exec ${ns}"
+ shift
- # If failure occurred during or before qemu start up, then we need
- # to clean this up ourselves.
- if [[ -e "${QEMU_PIDFILE}" ]]; then
- rm "${QEMU_PIDFILE}"
- fi
+ ${NS_EXEC} ssh -q -o UserKnownHostsFile=/dev/null -p ${SSH_HOST_PORT}
localhost $*
+
+ return $?
+}
+
+terminate_pidfiles() {
+ local pidfile
+
+ for pidfile in $@; do
+ if [[ -s "${pidfile}" ]]; then
+ pkill -SIGTERM -F ${pidfile} 2>&1 > /dev/null
+ fi
+
+ # If failure occurred during or before qemu start up, then we
need
+ # to clean this up ourselves.
+ if [[ -e "${pidfile}" ]]; then
+ rm "${pidfile}"
+ fi
+ done
}
check_args() {
@@ -189,7 +275,13 @@ handle_build() {
}
vm_start() {
+ local cid=$1
+ local ns=$2
+ local verify_boot=${3:-1}
+ local pidfile=${4:-}
+
local logfile=/dev/null
+ local qemu_opts=""
local verbose_opt=""
local kernel_opt=""
local qemu
@@ -201,35 +293,53 @@ vm_start() {
logfile=/dev/stdout
fi
+ qemu_opts="\
+ ${QEMU_OPTS} -device vhost-vsock-pci,guest-cid=${cid} \
+ --pidfile ${pidfile}
+ "
+
if [[ "${BUILD}" -eq 1 ]]; then
kernel_opt="${KERNEL_CHECKOUT}"
fi
- vng \
+ if [[ ! -z "${ns}" ]]; then
+ NS_EXEC="ip netns exec ${ns}"
+ fi
+
+ if [[ -z "${pidfile}" ]]; then
+ pidfile=$(mktemp $PIDFILE_TEMPLATE)
+ PIDFILES+=("${pidfile}")
+ fi
+
+ ${NS_EXEC} vng \
--run \
${kernel_opt} \
${verbose_opt} \
- --qemu-opts="${QEMU_OPTS}" \
+ --qemu-opts="${qemu_opts}" \
--qemu="${qemu}" \
--user root \
--append "${KERNEL_CMDLINE}" \
--rw &> ${logfile} &
- if ! timeout ${WAIT_TOTAL} \
- bash -c 'while [[ ! -s '"${QEMU_PIDFILE}"' ]]; do sleep 1;
done; exit 0'; then
- die "failed to boot VM"
- fi
+ timeout ${WAIT_TOTAL} \
+ bash -c 'while [[ ! -s '"${pidfile}"' ]]; do sleep 1; done;
exit 0'
}
vm_wait_for_ssh() {
+ local ns="${1}"
local i
i=0
- while true; do
+ while [[ true ]]; do
if [[ ${i} -gt ${WAIT_PERIOD_MAX} ]]; then
die "Timed out waiting for guest ssh"
fi
- if vm_ssh -- true; then
+ if [[ ! -z "${ns}" ]]; then
+ vm_ssh_ns "${ns}" -- true
+ else
+ vm_ssh -- true
+ fi
+ if [[ $? -eq 0 ]]; then
break
fi
i=$(( i + 1 ))
@@ -262,8 +372,9 @@ wait_for_listener()
vm_wait_for_listener() {
local port=$1
+ local host_ns=$2
- vm_ssh <<EOF
+ vm_ssh_ns "${host_ns}" <<EOF
$(declare -f wait_for_listener)
wait_for_listener ${port} ${WAIT_PERIOD} ${WAIT_PERIOD_MAX}
EOF
@@ -271,6 +382,17 @@ EOF
host_wait_for_listener() {
wait_for_listener "${TEST_HOST_PORT_LISTENER}" "${WAIT_PERIOD}"
"${WAIT_PERIOD_MAX}"
+ wait_for_listener ${TEST_HOST_PORT_LISTENER} ${WAIT_PERIOD}
${WAIT_PERIOD_MAX}
+}
+
+host_ns_wait_for_listener() {
+ local ns="${1}"
+ local port="${2}"
+
+ ip netns exec "${ns}" bash <<-EOF
+ $(declare -f wait_for_listener)
+ wait_for_listener ${port} ${WAIT_PERIOD} ${WAIT_PERIOD_MAX}
+ EOF
}
__log_stdin() {
@@ -331,7 +453,7 @@ test_vm_server_host_client() {
${VSOCK_TEST} \
--mode=client \
--control-host=127.0.0.1 \
- --peer-cid="${VSOCK_CID}" \
+ --peer-cid="${DEFAULT_CID}" \
--control-port="${TEST_HOST_PORT}" 2>&1 | log_host "${testname}"
return $?
@@ -343,7 +465,7 @@ test_vm_client_host_server() {
${VSOCK_TEST} \
--mode "server" \
--control-port "${TEST_HOST_PORT_LISTENER}" \
- --peer-cid "${VSOCK_CID}" 2>&1 | log_host "${testname}" &
+ --peer-cid "${DEFAULT_CID}" 2>&1 | log_host "${testname}" &
host_wait_for_listener
@@ -376,6 +498,309 @@ test_vm_loopback() {
return $?
}
+add_namespaces() {
+ local init=${1:-0}
+
+ for mode in "${MODES[@]}"; do
+ if ! ip netns add "${mode}"; then
+ return ${KSFT_FAIL}
+ fi
+
+ # e.g., global-2, local-2, mixed-2
+ if ! ip netns add "${mode}-2"; then
+ return ${KSFT_FAIL}
+ fi
+
+ if [[ ${init} -eq 1 ]]; then
+ ns_set_mode "${mode}" "${mode}"
+ ns_set_mode "${mode}-2" "${mode}"
+
+ # we need lo for qemu port forwarding
+ ip netns exec "${mode}" ip link set dev lo up
+ ip netns exec "${mode}-2" ip link set dev lo up
+ fi
+ done
+ return 0
+}
+
+del_namespaces() {
+ for mode in "${MODES[@]}"; do
+ ip netns del "${mode}"
+ ip netns del "${mode}-2"
+ done &>/dev/null
+}
+
+ns_set_mode() {
+ local ns=$1
+ local mode=$2
+
+ echo "${mode}" \
+ | ip netns exec "${ns}" \
+ tee /proc/net/vsock_ns_mode &>/dev/null
+}
+
+setup_bridge() {
+ local ns0
+ local ns1
+ local addr1
+
+ ns0=$1
+ ns1=$2
+
+ ip link add veth0 type veth peer name veth1
+ ip link set veth0 netns "${ns0}"
+ ip link set veth1 netns "${ns1}"
+ ip netns exec "${ns0}" ip addr add 10.0.0.1/24 dev veth0
+ ip netns exec "${ns1}" ip addr add 10.0.0.2/24 dev veth1
+ ip netns exec "${ns0}" ip link set veth0 up
+ ip netns exec "${ns1}" ip link set veth1 up
+}
+
+teardown_bridge() {
+ local ns0="${1}"
+
+ # veth1 is implicitly destroyed with veth0
+ ip netns exec "${ns0}" ip link delete veth0
+}
+
+test_host_vsock_ns_mode() {
+ if ! add_namespaces; then
+ return ${KSFT_FAIL}
+ fi
+
+ for mode in "${MODES[@]}"; do
+ if ! ns_set_mode "${mode}" "${mode}"; then
+ del_namespaces
+ return ${KSFT_FAIL}
+ fi
+ done
+
+ if ! del_namespaces; then
+ return ${KSFT_FAIL}
+ fi
+}
+
+test_host_vsock_ns_mode_write_once() {
+ if ! add_namespaces; then
+ return ${KSFT_FAIL}
+ fi
+
+ for mode in "${MODES[@]}"; do
+ if ! ns_set_mode "${mode}" "${mode}"; then
+ del_namespaces
+ return ${KSFT_FAIL}
+ fi
+
+ # try setting back to global, should fail
+ if ns_set_mode "${mode}" "global"; then
+ del_namespaces
+ return ${KSFT_FAIL}
+ fi
+ done
+
+ if ! del_namespaces; then
+ return ${KSFT_FAIL}
+ fi
+}
+
+namespaces_can_boot_same_cid() {
+ local ns1=$1
+ local ns2=$2
+ local cid=20
+ local pidfile1
+ local pidfile2
+ local msg
+
+ if ! add_namespaces 1; then
+ return 1
+ fi
+
+ if [[ ${VERBOSE} -gt 0 ]]; then
+ echo "booting vm 1" | tap_prefix
+ fi
+
+ pidfile1=$(mktemp $PIDFILE_TEMPLATE)
+ PIDFILES+=("${pidfile1}")
+ vm_start ${cid} ${ns1} ${pidfile1}
+
+ if [[ ${VERBOSE} -gt 0 ]]; then
+ echo "booting vm 2" | tap_prefix
+ fi
+
+ pidfile2=$(mktemp $PIDFILE_TEMPLATE)
+ PIDFILES+=("${pidfile2}")
+ WAIT_TOTAL=30 vm_start ${cid} ${ns2} ${pidfile2}
+
+ rc=$?
+ if [[ $rc -eq 0 ]]; then
+ msg="successfully booted"
+ rc=0
+ else
+ msg="failed to boot"
+ rc=1
+ fi
+
+ if [[ ${VERBOSE} -gt 0 ]]; then
+ echo "vm 2 ${msg}" | tap_prefix
+ fi
+ if ! del_namespaces; then
+ echo "failed to delete namespaces" | tap_prefix
+ fi
+
+ terminate_pidfiles ${pidfile1} ${pidfile2}
+ return $rc
+}
+
+test_global_same_cid() {
+ if namespaces_can_boot_same_cid "global" "global-2"; then
+ return $KSFT_FAIL
+ fi
+
+ return $KSFT_PASS
+}
+
+test_local_global_same_cid() {
+ if namespaces_can_boot_same_cid "local" "global"; then
+ return $KSFT_PASS
+ fi
+
+ return $KSFT_FAIL
+}
+
+test_global_local_same_cid() {
+ if namespaces_can_boot_same_cid "global" "local"; then
+ return $KSFT_PASS
+ fi
+
+ return $KSFT_FAIL
+}
+
+test_local_same_cid() {
+ if namespaces_can_boot_same_cid "local" "local"; then
+ return $KSFT_FAIL
+ fi
+
+ return $KSFT_PASS
+}
+
+test_global_host_connect_global_vm() {
+ local testname="${FUNCNAME[0]#test_}"
+ local cid=${DEFAULT_CID}
+ local port=1234
+ local host_ns="global"
+ local host_ns2="global-2"
+
+ add_namespaces 1
+ setup_bridge "${host_ns}" "${host_ns2}"
+
+ # Start server in VM in namespace
+ if ! vm_start ${cid} "${host_ns}"; then
+ teardown_bridge "${host_ns}"
+ return $KSFT_FAIL
+ fi
+
+ vm_ssh_ns "${host_ns}" \
+ -- "${VSOCK_TEST}" \
+ --mode=server \
+ --control-port="${TEST_GUEST_PORT}" \
+ --peer-cid=2 \
+ 2>&1 | log_guest "${testname}" &
+ vm_wait_for_listener ${TEST_GUEST_PORT} "${host_ns}"
+
+ # Setup NS-to-NS "bridge"
+ ip netns exec "${host_ns}" socat TCP-LISTEN:${BRIDGE_PORT},fork \
+ TCP-CONNECT:localhost:${TEST_HOST_PORT} &
+ host_ns_wait_for_listener "${host_ns}" "${BRIDGE_PORT}"
+
+ ip netns exec "${host_ns2}" \
+ socat TCP:10.0.0.1:${BRIDGE_PORT}
TCP-LISTEN:${TEST_HOST_PORT},fork &
+ host_ns_wait_for_listener "${host_ns2}" "${TEST_HOST_PORT}"
+
+ # Start client in other namespace
+ ip netns exec "${host_ns2}" ${VSOCK_TEST} \
+ --mode=client \
+ --control-host=127.0.0.1 \
+ --peer-cid="${cid}" \
+ --control-port="${TEST_HOST_PORT}" 2>&1 | log_host "${testname}"
+ rc=$?
+
+ if [[ ! $rc -eq 0 ]]; then
+ return $KSFT_FAIL
+ fi
+
+ del_namespaces
+
+ return $KSFT_PASS
+}
+
+do_ns_vm_client_host_server_test() {
+ local testname="$1"
+ local host_ns="$2"
+ local host_ns2="$3"
+ local cid=${DEFAULT_CID}
+
+ # must not be same as qemu hostfwd port
+ local port=12345
+
+ add_namespaces 1
+ setup_bridge "${host_ns}" "${host_ns2}"
+
+ if ! vm_start ${cid} "${host_ns}"; then
+ teardown_bridge "${host_ns}"
+ return $KSFT_FAIL
+ fi
+
+ ip netns exec "${host_ns2}" ${VSOCK_TEST} \
+ --mode=server \
+ --peer-cid="${cid}" \
+ --control-port="${port}" 2>&1 | log_host "${testname}" &
+
+ host_ns_wait_for_listener "${host_ns2}" "${port}"
+
+ ip netns exec "${host_ns2}" \
+ socat TCP-LISTEN:${BRIDGE_PORT},bind=10.0.0.2,fork \
+ TCP:localhost:${port} &
+
+ host_ns_wait_for_listener "${host_ns2}" "${BRIDGE_PORT}"
+
+ ip netns exec "${host_ns}" socat TCP-LISTEN:${port},fork \
+ TCP-CONNECT:10.0.0.2:${BRIDGE_PORT} &
+
+ host_ns_wait_for_listener "${host_ns}" "${port}"
+
+ vm_ssh_ns "${host_ns}" \
+ -- "${VSOCK_TEST}" \
+ --mode=client \
+ --control-host=10.0.2.2 \
+ --control-port="${port}" \
+ --peer-cid=2 \
+ 2>&1 | log_guest "${testname}"
+
+ if [[ ! $? -eq 0 ]]; then
+ return $KSFT_FAIL
+ fi
+
+ del_namespaces
+
+ return $KSFT_PASS
+}
+
+test_global_vm_connect_global_host() {
+ local testname="${FUNCNAME[0]#test_}"
+ local host_ns="global"
+ local host_ns2="global-2"
+
+ do_ns_vm_client_host_server_test ${testname} ${host_ns} ${host_ns2}
+}
+
+test_global_vm_connect_mixed_host() {
+ local testname="${FUNCNAME[0]#test_}"
+ local host_ns="global"
+ local host_ns2="mixed"
+
+ do_ns_vm_client_host_server_test ${testname} ${host_ns} ${host_ns2}
+}
+
run_test() {
local host_oops_cnt_before
local host_warn_cnt_before
@@ -421,7 +846,40 @@ run_test() {
rc=$KSFT_FAIL
fi
- return "${rc}"
+ check_result "${rc}"
+}
+
+needs_setup() {
+ local tname
+
+ tname="$1"
+
+ for testname in ${NEEDS_SETUP[@]}; do
+ if [[ "${tname}" == "${testname}" ]]; then
+ return 1
+ fi
+ done
+
+ return 0
+}
+
+check_result() {
+ local rc
+
+ rc=$1
+
+ if [[ ${rc} -eq $KSFT_PASS ]]; then
+ cnt_pass=$(( cnt_pass + 1 ))
+ echo "ok ${cnt_total} ${arg}"
+ elif [[ ${rc} -eq $KSFT_SKIP ]]; then
+ cnt_skip=$(( cnt_skip + 1 ))
+ echo "ok ${cnt_total} ${arg} # SKIP"
+ elif [[ ${rc} -eq $KSFT_FAIL ]]; then
+ cnt_fail=$(( cnt_fail + 1 ))
+ echo "not ok ${cnt_total} ${arg} # exit=$rc"
+ fi
+
+ cnt_total=$(( cnt_total + 1 ))
}
QEMU="qemu-system-$(uname -m)"
@@ -452,29 +910,36 @@ handle_build
echo "1..${#ARGS[@]}"
-log_setup "Booting up VM"
-vm_start
-vm_wait_for_ssh
-log_setup "VM booted up"
-
cnt_pass=0
cnt_fail=0
cnt_skip=0
cnt_total=0
+setup_done=0
+
+pidfile=""
+for arg in ${ARGS[@]}; do
+ if needs_setup "${arg}"; then
+ if [[ -z "${pidfile}" ]]; then
+ pidfile=$(mktemp $PIDFILE_TEMPLATE)
+ log_setup "Booting up VM"
+ vm_start "${DEFAULT_CID}" "" "${pidfile}"
+ vm_wait_for_ssh
+ log_setup "VM booted up"
+ fi
+
+ run_test "${arg}"
+ fi
+done
+
+if [[ ! -z "${pidfile}" ]]; then
+ log_setup "VM terminate"
+ terminate_pidfiles "${pidfile}"
+fi
+
for arg in "${ARGS[@]}"; do
- run_test "${arg}"
- rc=$?
- if [[ ${rc} -eq $KSFT_PASS ]]; then
- cnt_pass=$(( cnt_pass + 1 ))
- echo "ok ${cnt_total} ${arg}"
- elif [[ ${rc} -eq $KSFT_SKIP ]]; then
- cnt_skip=$(( cnt_skip + 1 ))
- echo "ok ${cnt_total} ${arg} # SKIP"
- elif [[ ${rc} -eq $KSFT_FAIL ]]; then
- cnt_fail=$(( cnt_fail + 1 ))
- echo "not ok ${cnt_total} ${arg} # exit=$rc"
+ if ! needs_setup "${arg}"; then
+ run_test "${arg}"
fi
- cnt_total=$(( cnt_total + 1 ))
done
echo "SUMMARY: PASS=${cnt_pass} SKIP=${cnt_skip} FAIL=${cnt_fail}"
--
2.47.1
