This patch includes changes made earlier by David in the ovs branch to cache the dpdk builds.
Co-authored-by: David Marchand <david.march...@redhat.com> Signed-off-by: David Marchand <david.march...@redhat.com> Signed-off-by: Eelco Chaudron <echau...@redhat.com> --- v2: Replaced 'sleep 1' with '' after consulting with Dumitru. Note that I ran the full GitHub ci 20x for dpdk on the v1 patchset. For v2, I did 10 runs of only the changed/fixed test. .ci/ci.sh | 12 ++++++- .ci/dpdk-build.sh | 54 ++++++++++++++++++++++++++++++ .ci/dpdk-prepare.sh | 11 ++++++ .ci/linux-build.sh | 49 ++++++++++++++++++++++++++- .github/workflows/test.yml | 80 ++++++++++++++++++++++++++++++++++++++++++++ Makefile.am | 2 + tests/system-ovn.at | 2 + 7 files changed, 207 insertions(+), 3 deletions(-) create mode 100755 .ci/dpdk-build.sh create mode 100755 .ci/dpdk-prepare.sh diff --git a/.ci/ci.sh b/.ci/ci.sh index 90942bab6..10f11939c 100755 --- a/.ci/ci.sh +++ b/.ci/ci.sh @@ -16,6 +16,7 @@ OVN_PATH=${OVN_PATH:-$PWD} OVS_PATH=${OVS_PATH:-$OVN_PATH/ovs} +DPDK_PATH=${DPDK_PATH:-$OVN_PATH/dpdk-dir} CONTAINER_CMD=${CONTAINER_CMD:-podman} CONTAINER_WORKSPACE="/workspace" CONTAINER_WORKDIR="/workspace/ovn-tmp" @@ -80,6 +81,10 @@ function copy_sources_to_workdir() { && \ cp -a $CONTAINER_WORKSPACE/ovs/. $CONTAINER_WORKDIR/ovs \ && \ + rm -rf $CONTAINER_WORKDIR/dpdk-dir \ + && \ + cp -a $CONTAINER_WORKSPACE/dpdk-dir/. $CONTAINER_WORKDIR/dpdk-dir \ + && \ git config --global --add safe.directory $CONTAINER_WORKDIR " } @@ -95,7 +100,7 @@ function run_tests() { cd $CONTAINER_WORKDIR \ && \ ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \ - TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS \ + TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \ ./.ci/linux-build.sh " } @@ -148,12 +153,17 @@ if [ "$ARCH" = "aarch64" ]; then ASAN_OPTIONS="detect_leaks=0" fi +if [ -z "$DPDK" ]; then + mkdir -p "$DPDK_PATH" +fi + CONTAINER_ID="$($CONTAINER_CMD run --privileged -d \ --pids-limit=-1 \ --env ASAN_OPTIONS=$ASAN_OPTIONS \ -v /lib/modules/$(uname -r):/lib/modules/$(uname -r):ro \ -v $OVN_PATH:$CONTAINER_WORKSPACE/ovn:Z \ -v $OVS_PATH:$CONTAINER_WORKSPACE/ovs:Z \ + -v $DPDK_PATH:$CONTAINER_WORKSPACE/dpdk-dir:Z \ $IMAGE_NAME)" trap remove_container EXIT diff --git a/.ci/dpdk-build.sh b/.ci/dpdk-build.sh new file mode 100755 index 000000000..f44ac15b0 --- /dev/null +++ b/.ci/dpdk-build.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -o errexit +set -x + +function build_dpdk() +{ + local VERSION_FILE="dpdk-dir/cached-version" + local DPDK_VER=$1 + local DPDK_OPTS="" + + rm -rf dpdk-dir + + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then + git clone --single-branch $DPDK_GIT dpdk-dir -b "${DPDK_VER##refs/*/}" + pushd dpdk-dir + git log -1 --oneline + else + wget https://fast.dpdk.org/rel/dpdk-$1.tar.xz + tar xvf dpdk-$1.tar.xz > /dev/null + DIR_NAME=$(tar -tf dpdk-$1.tar.xz | head -1 | cut -f1 -d"/") + mv ${DIR_NAME} dpdk-dir + pushd dpdk-dir + fi + + # Switching to 'default' machine to make dpdk-dir cache usable on + # different CPUs. We can't be sure that all CI machines are exactly same. + DPDK_OPTS="$DPDK_OPTS -Dmachine=default" + + # Disable building DPDK unit tests. Not needed for OVS build or tests. + DPDK_OPTS="$DPDK_OPTS -Dtests=false" + + # Disable DPDK developer mode, this results in less build checks and less + # meson verbose outputs. + DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled" + + # OVS compilation and the "ovn-system-dpdk" unit tests (run in the CI) + # only depend on virtio/tap drivers. + # We can disable all remaining drivers to save compilation time. + DPDK_OPTS="$DPDK_OPTS -Denable_drivers=net/null,net/tap,net/virtio" + + # Install DPDK using prefix. + DPDK_OPTS="$DPDK_OPTS --prefix=$(pwd)/build" + + meson $DPDK_OPTS build + ninja -C build + ninja -C build install + + echo "Installed DPDK in $(pwd)" + popd + echo "${DPDK_VER}" > ${VERSION_FILE} +} + +build_dpdk $DPDK_VER diff --git a/.ci/dpdk-prepare.sh b/.ci/dpdk-prepare.sh new file mode 100755 index 000000000..f7e6215dd --- /dev/null +++ b/.ci/dpdk-prepare.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ev + +# Installing wheel separately because it may be needed to build some +# of the packages during dependency backtracking and pip >= 22.0 will +# abort backtracking on build failures: +# https://github.com/pypa/pip/issues/10655 +pip3 install --disable-pip-version-check --user wheel +pip3 install --disable-pip-version-check --user pyelftools +pip3 install --user 'meson==0.53.2' diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh index 907a0dc6c..5a79a52da 100755 --- a/.ci/linux-build.sh +++ b/.ci/linux-build.sh @@ -10,13 +10,54 @@ OVN_CFLAGS="" OPTS="$OPTS --enable-Werror" JOBS=${JOBS:-"-j4"} +function install_dpdk() +{ + local VERSION_FILE="dpdk-dir/cached-version" + local DPDK_LIB=$(pwd)/dpdk-dir/build/lib/x86_64-linux-gnu + + # Export the following path for pkg-config to find the .pc file. + export PKG_CONFIG_PATH=$DPDK_LIB/pkgconfig/:$PKG_CONFIG_PATH + + if [ ! -f "${VERSION_FILE}" ]; then + echo "Could not find DPDK in $(pwd)/dpdk-dir" + return 1 + fi + + # As we build inside a container we need to update the prefix. + sed -i -E "s|^prefix=.*|prefix=$(pwd)/dpdk-dir/build|" \ + "$DPDK_LIB/pkgconfig/libdpdk-libs.pc" + + # Update the library paths. + sudo ldconfig + echo "Found cached DPDK $(cat ${VERSION_FILE}) build in $(pwd)/dpdk-dir" +} + function configure_ovs() { + if [ "$DPDK" ]; then + # When DPDK is enabled, we need to build OVS twice. Once to have + # ovs-vswitchd with DPDK. But OVN does not like the OVS libraries to + # be compiled with DPDK enabled, hence we need a final clean build + # with this disabled. + install_dpdk + + pushd ovs + ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}" --with-dpdk=static \ + $* || { cat config.log; exit 1; } + make $JOBS || { cat config.log; exit 1; } + cp vswitchd/ovs-vswitchd vswitchd/ovs-vswitchd_with_dpdk + popd + fi + pushd ovs ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}" $* || \ - { cat config.log; exit 1; } + { cat config.log; exit 1; } make $JOBS || { cat config.log; exit 1; } popd + + if [ "$DPDK" ]; then + cp ovs/vswitchd/ovs-vswitchd_with_dpdk ovs/vswitchd/ovs-vswitchd + fi } function configure_ovn() @@ -104,6 +145,12 @@ if [ "$TESTSUITE" ]; then execute_system_tests "check-system-userspace" \ "system-userspace-testsuite.log" ;; + + "system-test-dpdk") + # The dpdk tests need huge page memory, so reserve some 2M pages. + sudo bash -c "echo 2048 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" + execute_system_tests "check-system-dpdk" "system-dpdk-testsuite.log" + ;; esac else configure_ovn $OPTS diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6e8eac8d1..fe2a14c40 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,11 +12,81 @@ concurrency: cancel-in-progress: true jobs: + build-dpdk: + env: + dependencies: gcc libnuma-dev ninja-build + CC: gcc + DPDK_GIT: https://dpdk.org/git/dpdk-stable + DPDK_VER: 22.11.1 + name: dpdk gcc + outputs: + dpdk_key: ${{ steps.gen_dpdk_key.outputs.key }} + runs-on: ubuntu-20.04 + timeout-minutes: 30 + + steps: + - name: checkout + uses: actions/checkout@v3 + + - name: update PATH + run: | + echo "$HOME/bin" >> $GITHUB_PATH + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: create ci signature file for the dpdk cache key + # This will collect most of DPDK related lines, so hash will be different + # if something changed in a way we're building DPDK including DPDK_VER. + # This also allows us to use cache from any branch as long as version + # and a way we're building DPDK stays the same. + run: | + grep -irE 'RTE_|DPDK|meson|ninja' .ci/dpdk-* > dpdk-ci-signature + grep -rwE 'DPDK_GIT|DPDK_VER' .github/ >> dpdk-ci-signature + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then + git ls-remote --heads $DPDK_GIT $DPDK_VER >> dpdk-ci-signature + fi + cat dpdk-ci-signature + + - name: generate ci DPDK key + id: gen_dpdk_key + env: + ci_key: ${{ hashFiles('dpdk-ci-signature') }} + run: echo 'key=dpdk-${{ env.ci_key }}' >> $GITHUB_OUTPUT + + - name: cache + id: dpdk_cache + uses: actions/cache@v3 + with: + path: dpdk-dir + key: ${{ steps.gen_dpdk_key.outputs.key }} + + - name: set up python + if: steps.dpdk_cache.outputs.cache-hit != 'true' + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: update APT cache + if: steps.dpdk_cache.outputs.cache-hit != 'true' + run: sudo apt update || true + - name: install common dependencies + if: steps.dpdk_cache.outputs.cache-hit != 'true' + run: sudo apt install -y ${{ env.dependencies }} + + - name: prepare + if: steps.dpdk_cache.outputs.cache-hit != 'true' + run: ./.ci/dpdk-prepare.sh + + - name: build + if: steps.dpdk_cache.outputs.cache-hit != 'true' + run: ./.ci/dpdk-build.sh + build-linux: + needs: build-dpdk env: IMAGE_NAME: ghcr.io/ovn-org/ovn-tests:ubuntu ARCH: ${{ matrix.cfg.arch }} CC: ${{ matrix.cfg.compiler }} + DPDK: ${{ matrix.cfg.dpdk }} LIBS: ${{ matrix.cfg.libs }} OPTS: ${{ matrix.cfg.opts }} TESTSUITE: ${{ matrix.cfg.testsuite }} @@ -43,6 +113,9 @@ jobs: - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: "-500" } - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: "501-1000" } - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: "1001-" } + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, test_range: "-100" } + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, test_range: "101-200" } + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, test_range: "201-" } - { compiler: gcc, testsuite: system-test-userspace, test_range: "-100" } - { compiler: gcc, testsuite: system-test-userspace, test_range: "101-200" } - { compiler: gcc, testsuite: system-test-userspace, test_range: "201-" } @@ -84,6 +157,13 @@ jobs: sort -V | tail -1) working-directory: ovs + - name: cache + if: matrix.cfg.dpdk != '' + uses: actions/cache@v3 + with: + path: dpdk-dir + key: ${{ needs.build-dpdk.outputs.dpdk_key }} + - name: build if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }} run: sudo -E ./.ci/ci.sh --archive-logs diff --git a/Makefile.am b/Makefile.am index f1bd72d94..27182c7bc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -87,6 +87,8 @@ EXTRA_DIST = \ NOTICE \ .cirrus.yml \ .ci/ci.sh \ + .ci/dpdk-build.sh \ + .ci/dpdk-prepare.sh \ .ci/linux-build.sh \ .ci/osx-build.sh \ .ci/osx-prepare.sh \ diff --git a/tests/system-ovn.at b/tests/system-ovn.at index 05c234edc..a3f8fdaa9 100644 --- a/tests/system-ovn.at +++ b/tests/system-ovn.at @@ -4267,7 +4267,7 @@ done # Enable IGMP snooping on sw1. ovn-nbctl set Logical_Switch sw1 other_config:mcast_querier="false" ovn-nbctl set Logical_Switch sw1 other_config:mcast_snoop="true" - +ovn-nbctl --wait=hv sync group_v4="239.0.1.68" # Inject IGMP Join for v4 group on sw1-p1. _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev