Xavier Simonart via dev <[email protected]> writes:
> When ovn is upgraded, ovn-controller is updated first on
> the compute nodes. Then ovn-northd and DB are upgraded.
> This patch tests whether the intermediate state (i.e. with
> ovn-controller being upgraded) works properly, running system
> tests from the base line (i.e. before the upgrade).
>
> Flow tables might change between releases.
> Hence this patch must take that into account by updating the (old)
> system tests with any updated table numbers.
> In some cases, (new) ovn-controller might change flows in existing
> tables, causing some 'upgrade' tests to fail.
> Such tests can be skipped using the TAG_TEST_NOT_UPGRADABLE tag.
>
> This patch upgrades the ci to run automatically some upgrade tests
> weekly. It also provides a shell script to run those tests locally.
>
> Upgrade-tests are run on push/pull only for LTS (24.03) and latest
> release (25.09) to avoid too long tests. Upgrades from other
> branches are run on schedule.
>
> This patch depends on patch [1] on branch-25.09.
>
> [1] "tests: Add new TAG_TEST_NOT_UPGRADABLE to some tests."
>
> Reported-at: https://issues.redhat.com/browse/FDP-1240
> Signed-off-by: Xavier Simonart <[email protected]>
> ---
Just an fyi - the CI system sometimes gives weird results when mixing
branch-XX.YY; what I mean is each time the script is going to apply a
patch, it will switch branches if that was specified. In this case, it
means the first patch in this series will switch to branch-25.09, and
the second patch will be applied on top of that branch as well.
> .ci/ci.sh | 1 +
> .ci/linux-build.sh | 95 +++++++++-
> .ci/linux-util.sh | 367 +++++++++++++++++++++++++++++++++++++
> .ci/test-upgrade-local.sh | 202 ++++++++++++++++++++
> .github/workflows/test.yml | 41 ++++-
> Makefile.am | 1 +
> tests/ovn-macros.at | 6 +
> 7 files changed, 697 insertions(+), 16 deletions(-)
> create mode 100755 .ci/test-upgrade-local.sh
>
> diff --git a/.ci/ci.sh b/.ci/ci.sh
> index 3640d3243..6798fbd78 100755
> --- a/.ci/ci.sh
> +++ b/.ci/ci.sh
> @@ -102,6 +102,7 @@ function run_tests() {
> ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \
> TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \
> RECHECK=$RECHECK UNSTABLE=$UNSTABLE TIMEOUT=$TIMEOUT \
> + BASE_VERSION=$BASE_VERSION \
> ./.ci/linux-build.sh
> "
> }
> diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh
> index 183833a16..890f4c59d 100755
> --- a/.ci/linux-build.sh
> +++ b/.ci/linux-build.sh
> @@ -1,7 +1,10 @@
> #!/bin/bash
>
> set -o errexit
> -set -x
> +# Enable debug output for CI, optional for local
> +if [ "${NO_DEBUG:-0}" = "0" ]; then
> + set -x
> +fi
>
> ARCH=${ARCH:-"x86_64"}
> USE_SPARSE=${USE_SPARSE:-"yes"}
> @@ -181,7 +184,7 @@ function run_system_tests()
>
> if ! sudo timeout -k 5m -v $TIMEOUT make $JOBS $type \
> TESTSUITEFLAGS="$TEST_RANGE" RECHECK=$RECHECK \
> - SKIP_UNSTABLE=$SKIP_UNSTABLE; then
> + SKIP_UNSTABLE=$SKIP_UNSTABLE UPGRADE_TEST=$UPGRADE_TEST; then
> # $log_file is necessary for debugging.
> cat tests/$log_file
> return 1
> @@ -190,19 +193,28 @@ function run_system_tests()
>
> function execute_system_tests()
> {
> - configure_ovn $OPTS
> - make $JOBS || { cat config.log; exit 1; }
> + local test_type=$1
> + local log_file=$2
> + local skip_build=$3
> +
> + # Only build if not already built (upgrade tests build separately)
> + if [ "$skip_build" != "yes" ]; then
> + configure_ovn $OPTS
> + make $JOBS || { cat config.log; exit 1; }
> + fi
>
> local stable_rc=0
> local unstable_rc=0
>
> - if ! SKIP_UNSTABLE=yes run_system_tests $@; then
> + if ! SKIP_UNSTABLE=yes UPGRADE_TEST=$UPGRADE_TEST \
> + run_system_tests $test_type $log_file; then
> stable_rc=1
> fi
>
> if [ "$UNSTABLE" ]; then
> if ! SKIP_UNSTABLE=no TEST_RANGE="-k unstable" RECHECK=yes \
> - run_system_tests $@; then
> + UPGRADE_TEST=$UPGRADE_TEST run_system_tests $test_type \
> + $log_file; then
> unstable_rc=1
> fi
> fi
> @@ -212,6 +224,72 @@ function execute_system_tests()
> fi
> }
>
> +function execute_upgrade_tests()
> +{
> + . .ci/linux-util.sh
> +
> + # Save current CI scripts (will be replaced by base version after
> checkout)
> + cp -rf .ci /tmp/ovn-upgrade-ci
> +
> + # Build current version
> + log "Building current version..."
> + mkdir -p logs
> + configure_ovn $OPTS >> logs/build-current.log 2>&1 || {
> + log "configure ovn failed - see config.log and
> logs/build-current.log"
> + exit 1
> + }
> + make $JOBS >> logs/build-current.log 2>&1 || {
> + log "building ovn failed - see logs/build-current.log"
> + exit 1
> + }
> +
> + ovn_upgrade_save_current_binaries
> +
> + # Checkout base version
> + ovn_upgrade_checkout_base "$BASE_VERSION" logs/git.log
> +
> + # Clean from current version
> + log "Cleaning build artifacts..."
> + make distclean >> logs/build-base.log 2>&1 || true
> + (cd ovs && make distclean >> ../logs/build-base.log 2>&1) || true
> +
> + # Apply test patches
> + ovn_upgrade_apply_tests_patches
> +
> + # Build base with patches
> + ovn_upgrade_patch_for_ovn_debug
> +
> + # Build (modified) base version
> + log "Building base version (with patched lflow.h)..."
> + configure_ovn $OPTS >> logs/build-base.log 2>&1 || {
> + log "configure ovn failed - see config.log and logs/build-base.log"
> + exit 1
> + }
> + make $JOBS >> logs/build-base.log 2>&1 || {
> + log "building ovn failed - see logs/build-base.log"
> + exit 1
> + }
> + ovn_upgrade_save_ovn_debug
> +
> + # Build (clean) base version
> + log "Rebuilding base version (clean lflow.h)..."
> + git checkout controller/lflow.h >> logs/git.log 2>&1
> + make $JOBS >> logs/build-base.log 2>&1 || {
> + log "building ovn failed - see logs/build-base.log"
> + exit 1
> + }
> +
> + # Restore binaries
> + ovn_upgrade_restore_binaries
> +
> + # Restore current CI scripts for test execution
> + cp -f /tmp/ovn-upgrade-ci/linux-build.sh .ci/linux-build.sh
> + cp -f /tmp/ovn-upgrade-ci/linux-util.sh .ci/linux-util.sh
> +
> + UPGRADE_TEST=yes execute_system_tests "check-kernel" \
> + "system-kmod-testsuite.log" "yes"
> +}
> +
> configure_$CC
>
> if [ "$TESTSUITE" ]; then
> @@ -238,6 +316,11 @@ if [ "$TESTSUITE" ]; then
> sudo bash -c "echo 2048 >
> /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
> execute_system_tests "check-system-dpdk" "system-dpdk-testsuite.log"
> ;;
> +
> + "upgrade-test")
> + execute_upgrade_tests
> + ;;
> +
> esac
> else
> configure_ovn $OPTS
> diff --git a/.ci/linux-util.sh b/.ci/linux-util.sh
> index b5bd1f8c9..a62ff2ea0 100755
> --- a/.ci/linux-util.sh
> +++ b/.ci/linux-util.sh
> @@ -49,3 +49,370 @@ function disable_apparmor()
> sudo aa-teardown || true
> sudo systemctl disable --now apparmor.service
> }
> +
> +function log() {
> + echo "[$(date '+%H:%M:%S')] $*"
> +}
> +
> +# ovn_upgrade_save_current_binaries
> +# Saves current version's binaries and schemas to /tmp/ovn-upgrade-binaries/
> +function ovn_upgrade_save_current_binaries()
> +{
> + mkdir -p /tmp/ovn-upgrade-binaries
> +
> + # New ovn-controller may generate OpenFlow flows with new actions that
> old
> + # OVS doesn't understand, so we need also to use new OVS.
> + # New ovs-vswitchd binary expects columns/tables defined in the current
> + # schema, so we also need to use new schemas.
> + files="controller/ovn-controller ovs/vswitchd/ovs-vswitchd
> + ovs/ovsdb/ovsdb-server ovs/utilities/ovs-vsctl
> + ovs/utilities/ovs-ofctl ovs/utilities/ovs-appctl
> + ovs/utilities/ovs-dpctl ovs/vswitchd/vswitch.ovsschema"
> + for file in $files; do
> + if [ ! -f "$file" ]; then
> + log "ERROR: $file not found"
> + return 1
> + fi
> + cp "$file" /tmp/ovn-upgrade-binaries/
> + done
> +
> + # In the upgrade scenario we use old ovn-northd and new ovn-controller.
> + # OFTABLES are defined through a combination of northd/northd.h and
> + # controller/lflow.h. Tests uses either (old) table numbers, table names
> + # (defined in ovn-macros) or ovn-debug.
> + #
> + # Extract OFCTL_* table defines from current lflow.h
> + if ! grep '^#define OFTABLE_' controller/lflow.h > \
> + /tmp/ovn-upgrade-ofctl-defines.h; then
> + log "No #define OFTABLE_ found in lflow.h"
> + return 1
> + fi
> +
> + # Extract OFTABLE m4 defines from current tests/ovn-macros.at
> + # These are used by tests to reference table numbers
> + # In old tests, there might be no OFTABLE_ in ovn-macros, so grep can
> fail.
> + grep '^m4_define(\[OFTABLE_' tests/ovn-macros.at > \
> + /tmp/ovn-upgrade-oftable-m4-defines.txt || true
> +
> + # Extract key table numbers for calculating shifts in hardcoded table
> + # references. OFTABLE_SAVE_INPORT is where normal (unshifted) tables
> + # resume.
> + LINE=$(grep "define OFTABLE_LOG_EGRESS_PIPELINE" controller/lflow.h)
> + NEW_LOG_EGRESS=$(echo "$LINE" | grep -oE '[0-9]+')
> + LINE=$(grep "define OFTABLE_SAVE_INPORT" controller/lflow.h)
> + NEW_SAVE_INPORT=$(echo "$LINE" | grep -oE '[0-9]+')
> + if [ -z "$NEW_LOG_EGRESS" ]; then
> + log "ERROR: Could not extract OFTABLE_LOG_EGRESS_PIPELINE value"
> + return 1
> + fi
> + if [ -z "$NEW_SAVE_INPORT" ]; then
> + log "ERROR: Could not extract OFTABLE_SAVE_INPORT value"
> + return 1
> + fi
> +
> + echo "$NEW_LOG_EGRESS" > /tmp/ovn-upgrade-new-log-egress.txt
> + echo "$NEW_SAVE_INPORT" > /tmp/ovn-upgrade-new-save-inport.txt
> +
> + echo ""
> + log "Saved current versions:"
> + log " ovn-controller:$(/tmp/ovn-upgrade-binaries/ovn-controller
> --version |
> + grep ovn-controller)"
> + log " SB DB schema:$(/tmp/ovn-upgrade-binaries/ovn-controller --version |
> + grep "SB DB Schema")"
> + log " ovs-vswitchd:$(/tmp/ovn-upgrade-binaries/ovs-vswitchd --version |
> + grep vSwitch)"
> +}
> +
> +# ovn_upgrade_checkout_base BASE_VERSION LOG_FILE
> +# Checks out base version from git
> +function ovn_upgrade_checkout_base()
> +{
> + local base_version=$1
> + local log_file=$2
> +
> + log "Checking out base version: $base_version"
> +
> + # Try to checkout directly first (might already exist locally)
> + if git checkout "$base_version" >> "$log_file" 2>&1; then
> + log "Using locally available $base_version"
> + else
> + # Not available locally, try to fetch it
> + log "Fetching $base_version from origin..."
> +
> + # Try as a tag first
> + if git fetch --depth=1 origin tag "$base_version" \
> + >> "$log_file" 2>&1; then
> + log "Fetched tag $base_version"
> +
> + # Try as a branch
> + elif git fetch --depth=1 origin "$base_version" \
> + >> "$log_file" 2>&1; then
> + log "Fetched branch $base_version"
> +
> + else
> + git fetch origin >> "$log_file" 2>&1 || true
> + log "Fetched all refs from origin"
> + fi
> +
> + # Try checkout
> + if git checkout "$base_version" >> "$log_file" 2>&1; then
> + log "Using $base_version from origin"
> + else
> + # origin might be a private repo w/o all branches.
> + # Try ovn-org as fallback.
> + log "Not in origin, fetching from ovn-org..."
> + git fetch https://github.com/ovn-org/ovn.git \
> + "$base_version:$base_version" >> "$log_file" 2>&1 || return 1
> + log "Fetched $base_version from ovn-org"
> + git checkout "$base_version" >> "$log_file" 2>&1 || return 1
> + fi
> + fi
> +
> + git submodule update --init >> "$log_file" 2>&1 || return 1
> +}
> +
> +# Patch base version's lflow.h with current OFTABLE table defines
> +# This ensures ovn-debug uses correct table numbers
> +function ovn_upgrade_patch_for_ovn_debug()
> +{
> + if [ -f /tmp/ovn-upgrade-ofctl-defines.h ] && \
> + [ -f controller/lflow.h ]; then
> + # Replace old OFCTL defines with current ones in one pass
> + awk '
> + !inserted && /^#define OFTABLE_/ {
> + system("cat /tmp/ovn-upgrade-ofctl-defines.h")
> + inserted = 1
> + }
> + /^#define OFTABLE_/ { next }
> + { print }
> + ' controller/lflow.h > controller/lflow.h.tmp
> +
> + mv controller/lflow.h.tmp controller/lflow.h
> + fi
> +}
> +
> +# ovn_upgrade_save_ovn_debug
> +# Saves ovn-debug binary built with current OFTABLE defines
> +# This creates a hybrid ovn-debug: current table numbers + base logical flow
> +# stages
> +function ovn_upgrade_save_ovn_debug()
> +{
> + log "Saving hybrid ovn-debug..."
> + cp utilities/ovn-debug /tmp/ovn-upgrade-binaries/ovn-debug
> +}
> +
> +# update_test old_first_table old_last_table shift test_file
> +# Update test tables in test_file, for old_first <= tables < old_last_table
> +function update_test()
> +{
> + test_file=$4
> + awk -v old_start=$1 \
> + -v old_end=$2 \
> + -v shift=$3 '
> + {
> + result = ""
> + rest = $0
> + # Process all table=NUMBER matches in the line
> + while (match(rest, /table *= *[0-9]+/)) {
> + # Save match position before calling match() again
> + pos = RSTART
> + len = RLENGTH
> +
> + # Add everything before the match
> + result = result substr(rest, 1, pos-1)
> +
> + # Extract the matched text and the number
> + matched = substr(rest, pos, len)
> + if (match(matched, /[0-9]+/)) {
> + num = substr(matched, RSTART, RLENGTH)
> + } else {
> + num = 0
> + }
> +
> + # Check if this table number needs updating
> + if (num >= old_start && num < old_end) {
> + result = result "table=" (num + shift)
> + } else {
> + result = result matched
> + }
> +
> + # Continue with the rest of the line (use saved pos/len)
> + rest = substr(rest, pos + len)
> + }
> + # Add any remaining text
> + print result rest
> + }' "$test_file" > "$test_file.tmp" && mv "$test_file.tmp" "$test_file"
> +}
> +
> +# ovn_upgrade_table_numbers_in_tests_patch: fix hardcoded table numbers in
> +# test files
> +function ovn_upgrade_table_numbers_in_tests_patch()
> +{
> + # Old tests (e.g., branch-24.03) have hardcoded numbers like "table=45"
> + # which refer to specific logical tables. When OFTABLE defines shift,
> + # these numbers must be updated.
> + # Example: v24.03.0 has OFTABLE_LOG_EGRESS_PIPELINE=42, so "table=45"
> + # means egress+3.
> + # In main, OFTABLE_LOG_EGRESS_PIPELINE=47, so it should become
> "table=50".
> + if [ ! -f /tmp/ovn-upgrade-new-log-egress.txt ] ||
> + [ ! -f /tmp/ovn-upgrade-new-save-inport.txt ]; then
> + log "WARNING: Table shift data not found, skipping hardcoded table \
> + number updates"
> + return
> + fi
> +
> + if [ ! -f controller/lflow.h ]; then
> + log "WARNING: controller/lflow.h not found, skipping hardcoded table
> \
> + number updates"
> + return
> + fi
> +
> + NEW_LOG_EGRESS=$(cat /tmp/ovn-upgrade-new-log-egress.txt)
> + NEW_SAVE_INPORT=$(cat /tmp/ovn-upgrade-new-save-inport.txt)
> +
> + # Get old values from base version's lflow.h (before we patched it)
> + LINE=$(grep "#define OFTABLE_LOG_EGRESS_PIPELINE" controller/lflow.h)
> + OLD_LOG_EGRESS=$(echo "$LINE" | grep -oE '[0-9]+')
> + LINE=$(grep "#define OFTABLE_SAVE_INPORT" controller/lflow.h)
> + OLD_SAVE_INPORT=$(echo "$LINE" | grep -oE '[0-9]+')
> +
> + if [ -z "$OLD_LOG_EGRESS" ] || [ -z "$OLD_SAVE_INPORT" ] || \
> + [ "$OLD_LOG_EGRESS" == "$NEW_LOG_EGRESS" ]; then
> + log "No change in tests files as old_log_egress=$OLD_LOG_EGRESS,
> + old_save_inport=$OLD_SAVE_INPORT and
> + new_log_egress=$NEW_LOG_EGRESS"
> + return
> + fi
> +
> + # Calculate the shift
> + SHIFT=$((NEW_LOG_EGRESS - OLD_LOG_EGRESS))
> +
> + log "Updating hardcoded table numbers in tests (shift: +$SHIFT for
> tables \
> + $OLD_LOG_EGRESS-$((OLD_SAVE_INPORT-1)))"
> +
> + # Update hardcoded table numbers in test files
> + for test_file in tests/system-ovn.at tests/system-ovn-kmod.at; do
> + if [ -f "$test_file" ]; then
> + log "Updating $test_file"
> + update_test "$OLD_LOG_EGRESS" "$OLD_SAVE_INPORT" "$SHIFT" \
> + "$test_file"
> + fi
> + done
> +}
> +
> +# ovn_upgrade_cleanup_sbox_patch: filter out expected schema warnings.
> +function ovn_upgrade_cleanup_sbox_patch()
> +{
> + cat << 'EOF' > /tmp/upgrade-schema-filter.patch
> +diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at
> +index a08252d50..a31dc87b4 100644
> +--- a/tests/ovn-macros.at
> ++++ b/tests/ovn-macros.at
> +@@ -98,6 +98,7 @@ m4_define([OVN_CLEANUP_SBOX],[
> + $error
> + /connection failed (No such file or directory)/d
> + /has no network name*/d
> ++ /OVN_Southbound database lacks/d
> + /receive tunnel port not found*/d
> + /Failed to locate tunnel to reach main chassis/d
> + /Transaction causes multiple rows.*MAC_Binding/d
> +diff --git a/tests/system-kmod-macros.at b/tests/system-kmod-macros.at
> +index 6f6670199..4bd1a2c90 100644
> +--- a/tests/system-kmod-macros.at
> ++++ b/tests/system-kmod-macros.at
> +@@ -45,7 +45,8 @@ m4_define([OVS_TRAFFIC_VSWITCHD_START],
> + # invoked. They can be used to perform additional cleanups such as name
> space
> + # removal.
> + m4_define([OVS_TRAFFIC_VSWITCHD_STOP],
> +- [OVS_VSWITCHD_STOP([$1])
> ++ [OVS_VSWITCHD_STOP([dnl
> ++$1";/OVN_Southbound database lacks/d"])
> + AT_CHECK([:; $2])
> + ])
> +
> +EOF
> +
> + # Try to apply schema filter patch. May fail on old OVN versions where
> + # OVN_CLEANUP_SBOX doesn't check errors - this is expected and okay.
> + # If patch fails for more recent OVN, then the test will fail due to the
> + # "OVN_Southbound database lacks".
> + patch -p1 < /tmp/upgrade-schema-filter.patch > /dev/null 2>&1 || true
> + rm -f /tmp/upgrade-schema-filter.patch
> +}
> +
> +# ovn_upgrade_oftable_ovn_macro_patch: update table numbers in ovn-macro
> +function ovn_upgrade_oftable_ovn_macro_patch()
> +{
> + # Patch base version's tests/ovn-macros.at with current OFTABLE m4
> defines
> + # This ensures tests use correct table numbers when checking flows
> + if [ -f /tmp/ovn-upgrade-oftable-m4-defines.txt ] &&
> + [ -f tests/ovn-macros.at ]; then
> + # Check if the base version has OFTABLE m4 defines
> + if grep -q '^m4_define(\[OFTABLE_' tests/ovn-macros.at; then
> + # Replace old m4_define OFTABLE statements with current ones
> + awk '
> + !inserted && /^m4_define\(\[OFTABLE_/ {
> + system("cat /tmp/ovn-upgrade-oftable-m4-defines.txt")
> + inserted = 1
> + }
> + /^m4_define\(\[OFTABLE_/ { next }
> + { print }
> + ' tests/ovn-macros.at > tests/ovn-macros.at.tmp
> +
> + mv tests/ovn-macros.at.tmp tests/ovn-macros.at
> + fi
> + fi
> +}
> +
> +# Applies patches to base version after second build:
> +# 1. Schema error patch (filters "OVN_Southbound database lacks" warnings)
> +# 2. OFTABLE m4 defines patch in tests/ovn-macros.at (for test table numbers)
> +# 3. Hardcoded table numbers patch in test files
> +function ovn_upgrade_apply_tests_patches()
> +{
> + log "Applying schema filter and table number patches..."
> + ovn_upgrade_table_numbers_in_tests_patch
> + ovn_upgrade_cleanup_sbox_patch
> + ovn_upgrade_oftable_ovn_macro_patch
> +}
> +
> +# ovn_upgrade_restore_binaries
> +#
> +# Replaces base version binaries with saved current versions:
> +# - ovn-controller (from current)
> +# - OVS binaries and schema (from current)
> +# - ovn-debug (hybrid: current OFTABLE + base logical stages)
> +function ovn_upgrade_restore_binaries()
> +{
> + log "Replacing binaries with current versions"
> +
> + # Replace OVN controller
> + cp /tmp/ovn-upgrade-binaries/ovn-controller controller/ovn-controller
> +
> + # Replace ovn-debug with hybrid version (built with current OFTABLE +
> base
> + # northd.h)
> + cp /tmp/ovn-upgrade-binaries/ovn-debug utilities/ovn-debug
> +
> + # Replace OVS binaries
> + cp /tmp/ovn-upgrade-binaries/ovs-vswitchd ovs/vswitchd/ovs-vswitchd
> + cp /tmp/ovn-upgrade-binaries/ovsdb-server ovs/ovsdb/ovsdb-server
> + cp /tmp/ovn-upgrade-binaries/ovs-vsctl ovs/utilities/ovs-vsctl
> + cp /tmp/ovn-upgrade-binaries/ovs-ofctl ovs/utilities/ovs-ofctl
> + cp /tmp/ovn-upgrade-binaries/ovs-appctl ovs/utilities/ovs-appctl
> + cp /tmp/ovn-upgrade-binaries/ovs-dpctl ovs/utilities/ovs-dpctl
> +
> + # Replace OVS schema (current binaries expect current schema)
> + cp /tmp/ovn-upgrade-binaries/vswitch.ovsschema \
> + ovs/vswitchd/vswitch.ovsschema
> +
> + echo ""
> + log "Verification - Current versions (from current patch):"
> + log " ovn-controller: $(controller/ovn-controller --version |
> + grep ovn-controller)"
> + log " SB DB Schema: $(controller/ovn-controller --version |
> + grep "SB DB Schema")"
> + log " ovs-vswitchd: $(ovs/vswitchd/ovs-vswitchd --version | grep
> vSwitch)"
> + log "Verification - Base versions (for compatibility testing):"
> + log " ovn-northd: $(northd/ovn-northd --version | grep ovn-northd)"
> + log " ovn-nbctl: $(utilities/ovn-nbctl --version | grep ovn-nbctl)"
> +}
> diff --git a/.ci/test-upgrade-local.sh b/.ci/test-upgrade-local.sh
> new file mode 100755
> index 000000000..3818ac465
> --- /dev/null
> +++ b/.ci/test-upgrade-local.sh
> @@ -0,0 +1,202 @@
> +#!/bin/bash
> +
> +set -e
> +
> +. "$(dirname $0)/linux-util.sh"
> +
> +BASE_VERSION="${BASE_VERSION:-branch-24.03}"
> +TEST_RANGE="${TEST_RANGE:-1-}"
> +KEEPALIVE_INT=50
> +
> +CLEANUP_DONE=0
> +TEST_STATUS=1
> +
> +usage() {
> + cat << EOF
> +Usage: $0 [options]
> +
> +Test OVN upgrade compatibility without GitHub.
> +
> +Options:
> + -b, --base-version VERSION Base version to test (default: branch-24.03)
> + -t, --test-range RANGE Test range to run (default: 1-)
> + Examples: -100, 101-, 55
> + -h, --help Show this help message
> +
> +Environment Variables:
> + BASE_VERSION Same as --base-version
> + TEST_RANGE Same as --test-range
> +
> +Examples:
> + # Test against branch-24.03 with all tests
> + $0
> +
> + # Test against specific version
> + $0 --base-version v24.03.0
> +
> + # Test specific test range
> + $0 --test-range 101-200
> +EOF
> +}
> +
> +# Parse command line arguments
> +while [[ $# -gt 0 ]]; do
> + case $1 in
> + -b|--base-version)
> + BASE_VERSION="$2"
> + shift 2
> + ;;
> + -t|--test-range)
> + TEST_RANGE="$2"
> + shift 2
> + ;;
> + -h|--help)
> + usage
> + exit 0
> + ;;
> + *)
> + echo "Unknown option: $1"
> + usage
> + exit 1
> + ;;
> + esac
> +done
> +
> +log "========================================"
> +log "OVN Upgrade Test"
> +log "Base version: $BASE_VERSION"
> +log "Test range: $TEST_RANGE"
> +
> +# Check if we're in the OVN repository root
> +if [ ! -f "configure.ac" ] || ! grep -q "ovn" configure.ac; then
> + log "Error: This script must be run from the OVN repository root"
> + exit 1
> +fi
> +
> +start_sudo_keepalive() {
> + (while true; do sudo -n true; sleep "$KEEPALIVE_INT"; done) 2>/dev/null &
> + SUDO_KEEPALIVE_PID=$!
> + if ! kill -0 $SUDO_KEEPALIVE_PID 2>/dev/null; then
> + log "ERROR: sudo keepalive failed to start"
> + exit 1
> + fi
> +}
> +
> +stop_sudo_keepalive() {
> + if [ -n "$SUDO_KEEPALIVE_PID" ]; then
> + kill $SUDO_KEEPALIVE_PID 2>/dev/null || true
> + fi
> +}
> +
> +# Cleanup function - always runs on exit
> +cleanup() {
> + if [ $CLEANUP_DONE -eq 1 ]; then
> + return
> + fi
> + CLEANUP_DONE=1
> +
> + stop_sudo_keepalive
> +
> + echo
> + log "Cleaning up..."
> + log "Restoring modified files..."
> + # Restore OVN test files
> + git checkout tests/ovn-macros.at tests/system-kmod-macros.at \
> + tests/system-ovn.at tests/system-ovn-kmod.at \
> + >> logs/git.log 2>&1 || true
> + # Restore OVS submodule files
> + (cd ovs && git checkout vswitchd/vswitch.ovsschema \
> + >> ../logs/git.log 2>&1 || true)
> + # Restore CI scripts (may have been replaced during upgrade test)
> + git checkout .ci/linux-build.sh .ci/linux-util.sh \
> + >> logs/git.log 2>&1 || true
> +
> + log "Restoring original branch/commit..."
> + # If we were on a branch, restore to it; otherwise restore to commit
> + if [ "$CURRENT_BRANCH" != "HEAD" ]; then
> + if ! git checkout "$CURRENT_BRANCH" >> logs/git.log 2>&1; then
> + log "WARNING: Failed to restore branch $CURRENT_BRANCH" >&2
> + fi
> + else
> + # We were in detached HEAD state, restore to the commit
> + if ! git checkout "$CURRENT_COMMIT" >> logs/git.log 2>&1; then
> + log "WARNING: Failed to restore commit $CURRENT_COMMIT" >&2
> + fi
> + fi
> +
> + log "Updating submodules..."
> + git submodule update --init >> logs/git.log 2>&1 || true
> + log "Restored to: $CURRENT_BRANCH ($CURRENT_COMMIT)"
> +
> + # Cleanup temporary files
> + rm -rf /tmp/ovn-upgrade-binaries /tmp/ovn-upgrade-ci
> + rm -f /tmp/ovn-upgrade-ofctl-defines.h
> + rm -f /tmp/ovn-upgrade-oftable-m4-defines.txt
> + rm -f /tmp/ovn-upgrade-new-log-egress.txt
> + rm -f /tmp/ovn-upgrade-new-save-inport.txt
> +}
> +
> +trap cleanup EXIT INT TERM QUIT HUP
> +
> +# Request sudo credentials early
> +if ! sudo -nv 2>/dev/null; then
> + log "This script requires sudo for running system tests."
> + log "Please enter your password now:"
> + sudo -v || {
> + log "Error: sudo authentication failed"
> + exit 1
> + }
> +fi
> +
> +start_sudo_keepalive
> +
> +# Save current branch/commit
> +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
> +CURRENT_COMMIT=$(git rev-parse HEAD)
> +log "Current branch: $CURRENT_BRANCH"
> +log "Current commit: $CURRENT_COMMIT"
> +echo
> +
> +# Check if working directory is clean
> +if ! git diff-index --quiet HEAD --; then
> + log "Warning: Working directory has uncommitted changes"
> + read -p "Continue anyway? (y/n) " -n 1 -r
> + echo
> + if [[ ! $REPLY =~ ^[Yy]$ ]]; then
> + exit 1
> + fi
> +fi
> +
> +# Create logs directory
> +mkdir -p logs
> +
> +# Export environment variables for linux-build.sh
> +export TESTSUITE="upgrade-test"
> +export BASE_VERSION="$BASE_VERSION"
> +export TEST_RANGE="$TEST_RANGE"
> +export JOBS="${JOBS:--j$(nproc 2>/dev/null || echo 4)}"
> +export CC="${CC:-gcc}"
> +export NO_DEBUG="${NO_DEBUG:-1}" # Disable verbose set -x output by default
> +export USE_SPARSE="${USE_SPARSE:-no}" # Disable sparse for local tests
> +
> +log "Running upgrade tests via linux-build.sh..."
> +echo
> +
> +# Run linux-build.sh which will call execute_upgrade_tests()
> +if ./.ci/linux-build.sh; then
> + echo
> + log "Upgrade test completed successfully"
> + TEST_STATUS=0
> +else
> + echo
> + log "Upgrade test failed - check logs"
> + TEST_STATUS=1
> +fi
> +
> +# Print summary
> +echo
> +log "Logs saved to:"
> +log " - logs/git.log"
> +log " - tests/system-kmod-testsuite.log"
> +
> +exit $TEST_STATUS
> diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
> index b6e461129..06b677f55 100644
> --- a/.github/workflows/test.yml
> +++ b/.github/workflows/test.yml
> @@ -94,7 +94,6 @@ jobs:
>
> name: linux ${{ join(matrix.cfg.*, ' ') }}
> runs-on: ubuntu-24.04
> -
> strategy:
> fail-fast: false
> matrix:
> @@ -123,30 +122,42 @@ jobs:
> - { compiler: clang, testsuite: system-test, sanitizers: sanitizers,
> test_range: "-100" }
> - { compiler: clang, testsuite: system-test, sanitizers: sanitizers,
> test_range: "101-200" }
> - { compiler: clang, testsuite: system-test, sanitizers: sanitizers,
> test_range: "201-", unstable: unstable }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-24.03, test_range: "-100" }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-24.03, test_range: "101-", unstable: unstable }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-24.09, test_range: "-100", run_on: schedule }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-24.09, test_range: "101-200", run_on: schedule }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-24.09, test_range: "201-", unstable: unstable, run_on: schedule }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-25.03, test_range: "-100", run_on: schedule }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-25.03, test_range: "101-200", run_on: schedule }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-25.03, test_range: "201-", unstable: unstable, run_on: schedule }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-25.09, test_range: "-100" }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-25.09, test_range: "101-200" }
> + - { compiler: gcc, testsuite: upgrade-test, base_version:
> branch-25.09, test_range: "201-", unstable: unstable }
> - { arch: x86, compiler: gcc, opts: --disable-ssl }
>
> steps:
> - name: system-level-dependencies
> - if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }}
> + if: ${{ (startsWith(matrix.cfg.testsuite, 'system-test') ||
> (matrix.cfg.testsuite == 'upgrade-test')) && (matrix.cfg.run_on != 'schedule'
> || github.event_name == 'schedule') }}
> run: |
> sudo apt update
> sudo apt -y install linux-modules-extra-$(uname -r)
>
> - name: checkout
> - if: github.event_name == 'push' || github.event_name == 'pull_request'
> + if: (github.event_name == 'push' || github.event_name ==
> 'pull_request' || matrix.cfg.testsuite == 'upgrade-test') &&
> (matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule')
> uses: actions/checkout@v4
> with:
> submodules: recursive
> + fetch-depth: ${{ matrix.cfg.testsuite == 'upgrade-test' && 0 || 1 }}
>
> - # For weekly runs, don't update submodules
> + # For weekly runs (no upgrade-tests), don't update submodules
> - name: checkout without submodule
> - if: github.event_name == 'schedule'
> + if: github.event_name == 'schedule' && matrix.cfg.testsuite !=
> 'upgrade-test'
> uses: actions/checkout@v4
>
> - # Weekly runs test using the tip of the most recent stable OVS branch
> + # Weekly runs (no upgrade-tests) test using the tip of the most recent
> stable OVS branch
> # instead of the submodule.
> - name: checkout OVS
> - if: github.event_name == 'schedule'
> + if: github.event_name == 'schedule' && matrix.cfg.testsuite !=
> 'upgrade-test'
> uses: actions/checkout@v4
> with:
> repository: 'openvswitch/ovs'
> @@ -154,7 +165,7 @@ jobs:
> path: 'ovs'
>
> - name: checkout OVS most recent stable branch.
> - if: github.event_name == 'schedule'
> + if: github.event_name == 'schedule' && matrix.cfg.testsuite !=
> 'upgrade-test'
> run: |
> git checkout \
> $(git branch -a -l '*branch-*' | sed 's/remotes\/origin\///' | \
> @@ -162,16 +173,19 @@ jobs:
> working-directory: ovs
>
> - name: Fix /etc/hosts file
> + if: matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule'
> run: |
> . .ci/linux-util.sh
> fix_etc_hosts
>
> - name: Disable apparmor
> + if: matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule'
> run: |
> . .ci/linux-util.sh
> disable_apparmor
>
> - name: image cache
> + if: matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule'
> id: image_cache
> uses: actions/cache@v4
> with:
> @@ -179,17 +193,24 @@ jobs:
> key: ${{ github.sha }}/${{ github.event_name }}
>
> - name: load image
> + if: matrix.cfg.run_on != 'schedule' || github.event_name == 'schedule'
> run: |
> sudo podman load -i /tmp/image.tar
> podman load -i /tmp/image.tar
> rm -rf /tmp/image.tar
>
> + # Set BASE_VERSION env var for upgrade tests
> + - name: Set upgrade test env
> + if: matrix.cfg.testsuite == 'upgrade-test' && (matrix.cfg.run_on !=
> 'schedule' || github.event_name == 'schedule')
> + run: echo "BASE_VERSION=${{ matrix.cfg.base_version }}" >> $GITHUB_ENV
> +
> + # Regular build steps
> - name: build
> - if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }}
> + if: ${{ (startsWith(matrix.cfg.testsuite, 'system-test') ||
> (matrix.cfg.testsuite == 'upgrade-test')) && (matrix.cfg.run_on != 'schedule'
> || github.event_name == 'schedule') }}
> run: sudo -E ./.ci/ci.sh --archive-logs --timeout=2h
>
> - name: build
> - if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') }}
> + if: ${{ !startsWith(matrix.cfg.testsuite, 'system-test') &&
> matrix.cfg.testsuite != 'upgrade-test' && (matrix.cfg.run_on != 'schedule' ||
> github.event_name == 'schedule') }}
> run: ./.ci/ci.sh --archive-logs --timeout=2h
>
> - name: upload logs on failure
> diff --git a/Makefile.am b/Makefile.am
> index 3ad2077b3..e57bfb297 100644
> --- a/Makefile.am
> +++ b/Makefile.am
> @@ -91,6 +91,7 @@ EXTRA_DIST = \
> .ci/linux-util.sh \
> .ci/osx-build.sh \
> .ci/osx-prepare.sh \
> + .ci/test-upgrade-local.sh \
> .ci/ovn-kubernetes/Dockerfile \
> .ci/ovn-kubernetes/prepare.sh \
> .ci/ovn-kubernetes/custom.patch \
> diff --git a/tests/ovn-macros.at b/tests/ovn-macros.at
> index 1c981e5a8..834ed60be 100644
> --- a/tests/ovn-macros.at
> +++ b/tests/ovn-macros.at
> @@ -1520,6 +1520,12 @@ m4_define([TAG_UNSTABLE], [
> AT_SKIP_IF([test X"$SKIP_UNSTABLE" = Xyes])
> ])
>
> +# TAG_TEST_NOT_UPGRADABLE tag indicates that the test would fail
> +# "upgrade" test (i.e. running old ovn-northd and new ovn-controller)
> +m4_define([TAG_TEST_NOT_UPGRADABLE], [
> + AT_SKIP_IF([test X"$UPGRADE_TEST" = Xyes])
> +])
> +
> m4_define([OVN_CHECK_SCAPY_EDNS_CLIENT_SUBNET_SUPPORT],
> [
> AT_SKIP_IF([test $HAVE_SCAPY = no])
_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
