On 4 Jul 2023, at 11:00, Ales Musil wrote:
> On Mon, Jun 19, 2023 at 11:32 AM Eelco Chaudron <echau...@redhat.com> wrote:
>
>> This patch includes changes made earlier by David in the
>> ovs branch to cache the dpdk builds.
>>
>> Co-authored-by: David Marchand <david.march...@redhat.com>
>> Signed-off-by: David Marchand <david.march...@redhat.com>
>> Signed-off-by: Eelco Chaudron <echau...@redhat.com>
>> ---
>>
>> v2: Replaced 'sleep 1' with '' after consulting with Dumitru.
>> v3: No changes
>>
>> Note that I ran the full GitHub ci 20x on the v1 patchset.
>> For v2, I did 10 runs of only the changed/fixed test. For v3,
>> I only did 1 run.
>>
>
> Hi Eelco,
>
> I have one comment down below.
>
>
>>
>> .ci/ci.sh | 12 ++++++-
>> .ci/dpdk-build.sh | 54 ++++++++++++++++++++++++++++++
>> .ci/dpdk-prepare.sh | 11 ++++++
>> .ci/linux-build.sh | 49 ++++++++++++++++++++++++++-
>> .github/workflows/test.yml | 80
>> ++++++++++++++++++++++++++++++++++++++++++++
>> Makefile.am | 2 +
>> tests/system-ovn.at | 2 +
>> 7 files changed, 207 insertions(+), 3 deletions(-)
>> create mode 100755 .ci/dpdk-build.sh
>> create mode 100755 .ci/dpdk-prepare.sh
>>
>> diff --git a/.ci/ci.sh b/.ci/ci.sh
>> index 90942bab6..10f11939c 100755
>> --- a/.ci/ci.sh
>> +++ b/.ci/ci.sh
>> @@ -16,6 +16,7 @@
>>
>> OVN_PATH=${OVN_PATH:-$PWD}
>> OVS_PATH=${OVS_PATH:-$OVN_PATH/ovs}
>> +DPDK_PATH=${DPDK_PATH:-$OVN_PATH/dpdk-dir}
>> CONTAINER_CMD=${CONTAINER_CMD:-podman}
>> CONTAINER_WORKSPACE="/workspace"
>> CONTAINER_WORKDIR="/workspace/ovn-tmp"
>> @@ -80,6 +81,10 @@ function copy_sources_to_workdir() {
>> && \
>> cp -a $CONTAINER_WORKSPACE/ovs/. $CONTAINER_WORKDIR/ovs \
>> && \
>> + rm -rf $CONTAINER_WORKDIR/dpdk-dir \
>> + && \
>> + cp -a $CONTAINER_WORKSPACE/dpdk-dir/. $CONTAINER_WORKDIR/dpdk-dir
>> \
>> + && \
>> git config --global --add safe.directory $CONTAINER_WORKDIR
>> "
>> }
>> @@ -95,7 +100,7 @@ function run_tests() {
>> cd $CONTAINER_WORKDIR \
>> && \
>> ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \
>> - TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS \
>> + TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \
>> ./.ci/linux-build.sh
>> "
>> }
>> @@ -148,12 +153,17 @@ if [ "$ARCH" = "aarch64" ]; then
>> ASAN_OPTIONS="detect_leaks=0"
>> fi
>>
>> +if [ -z "$DPDK" ]; then
>> + mkdir -p "$DPDK_PATH"
>> +fi
>> +
>> CONTAINER_ID="$($CONTAINER_CMD run --privileged -d \
>> --pids-limit=-1 \
>> --env ASAN_OPTIONS=$ASAN_OPTIONS \
>> -v /lib/modules/$(uname -r):/lib/modules/$(uname -r):ro \
>> -v $OVN_PATH:$CONTAINER_WORKSPACE/ovn:Z \
>> -v $OVS_PATH:$CONTAINER_WORKSPACE/ovs:Z \
>> + -v $DPDK_PATH:$CONTAINER_WORKSPACE/dpdk-dir:Z \
>> $IMAGE_NAME)"
>> trap remove_container EXIT
>>
>> diff --git a/.ci/dpdk-build.sh b/.ci/dpdk-build.sh
>> new file mode 100755
>> index 000000000..f44ac15b0
>> --- /dev/null
>> +++ b/.ci/dpdk-build.sh
>> @@ -0,0 +1,54 @@
>> +#!/bin/bash
>> +
>> +set -o errexit
>> +set -x
>> +
>> +function build_dpdk()
>> +{
>> + local VERSION_FILE="dpdk-dir/cached-version"
>> + local DPDK_VER=$1
>> + local DPDK_OPTS=""
>> +
>> + rm -rf dpdk-dir
>> +
>> + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then
>> + git clone --single-branch $DPDK_GIT dpdk-dir -b
>> "${DPDK_VER##refs/*/}"
>> + pushd dpdk-dir
>> + git log -1 --oneline
>> + else
>> + wget https://fast.dpdk.org/rel/dpdk-$1.tar.xz
>> + tar xvf dpdk-$1.tar.xz > /dev/null
>> + DIR_NAME=$(tar -tf dpdk-$1.tar.xz | head -1 | cut -f1 -d"/")
>> + mv ${DIR_NAME} dpdk-dir
>> + pushd dpdk-dir
>> + fi
>> +
>> + # Switching to 'default' machine to make dpdk-dir cache usable on
>> + # different CPUs. We can't be sure that all CI machines are exactly
>> same.
>> + DPDK_OPTS="$DPDK_OPTS -Dmachine=default"
>> +
>> + # Disable building DPDK unit tests. Not needed for OVS build or tests.
>> + DPDK_OPTS="$DPDK_OPTS -Dtests=false"
>> +
>> + # Disable DPDK developer mode, this results in less build checks and
>> less
>> + # meson verbose outputs.
>> + DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled"
>> +
>> + # OVS compilation and the "ovn-system-dpdk" unit tests (run in the CI)
>> + # only depend on virtio/tap drivers.
>> + # We can disable all remaining drivers to save compilation time.
>> + DPDK_OPTS="$DPDK_OPTS -Denable_drivers=net/null,net/tap,net/virtio"
>> +
>> + # Install DPDK using prefix.
>> + DPDK_OPTS="$DPDK_OPTS --prefix=$(pwd)/build"
>> +
>> + meson $DPDK_OPTS build
>> + ninja -C build
>> + ninja -C build install
>> +
>> + echo "Installed DPDK in $(pwd)"
>> + popd
>> + echo "${DPDK_VER}" > ${VERSION_FILE}
>> +}
>> +
>> +build_dpdk $DPDK_VER
>> diff --git a/.ci/dpdk-prepare.sh b/.ci/dpdk-prepare.sh
>> new file mode 100755
>> index 000000000..f7e6215dd
>> --- /dev/null
>> +++ b/.ci/dpdk-prepare.sh
>> @@ -0,0 +1,11 @@
>> +#!/bin/bash
>> +
>> +set -ev
>> +
>> +# Installing wheel separately because it may be needed to build some
>> +# of the packages during dependency backtracking and pip >= 22.0 will
>> +# abort backtracking on build failures:
>> +# https://github.com/pypa/pip/issues/10655
>> +pip3 install --disable-pip-version-check --user wheel
>> +pip3 install --disable-pip-version-check --user pyelftools
>> +pip3 install --user 'meson==0.53.2'
>>
>
> Do we need this whole dpdk-prepare.sh? We have moved all python
> dependencies to
> py-requirements.txt [0]. pyelftools are already there so the only missing
> package would be meson.
The dpdk cash is built outside of the containers, so we need this.
>> diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh
>> index 907a0dc6c..5a79a52da 100755
>> --- a/.ci/linux-build.sh
>> +++ b/.ci/linux-build.sh
>> @@ -10,13 +10,54 @@ OVN_CFLAGS=""
>> OPTS="$OPTS --enable-Werror"
>> JOBS=${JOBS:-"-j4"}
>>
>> +function install_dpdk()
>> +{
>> + local VERSION_FILE="dpdk-dir/cached-version"
>> + local DPDK_LIB=$(pwd)/dpdk-dir/build/lib/x86_64-linux-gnu
>> +
>> + # Export the following path for pkg-config to find the .pc file.
>> + export PKG_CONFIG_PATH=$DPDK_LIB/pkgconfig/:$PKG_CONFIG_PATH
>> +
>> + if [ ! -f "${VERSION_FILE}" ]; then
>> + echo "Could not find DPDK in $(pwd)/dpdk-dir"
>> + return 1
>> + fi
>> +
>> + # As we build inside a container we need to update the prefix.
>> + sed -i -E "s|^prefix=.*|prefix=$(pwd)/dpdk-dir/build|" \
>> + "$DPDK_LIB/pkgconfig/libdpdk-libs.pc"
>> +
>> + # Update the library paths.
>> + sudo ldconfig
>> + echo "Found cached DPDK $(cat ${VERSION_FILE}) build in
>> $(pwd)/dpdk-dir"
>> +}
>> +
>> function configure_ovs()
>> {
>> + if [ "$DPDK" ]; then
>> + # When DPDK is enabled, we need to build OVS twice. Once to have
>> + # ovs-vswitchd with DPDK. But OVN does not like the OVS libraries
>> to
>> + # be compiled with DPDK enabled, hence we need a final clean build
>> + # with this disabled.
>> + install_dpdk
>> +
>> + pushd ovs
>> + ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}"
>> --with-dpdk=static \
>> + $* || { cat config.log; exit 1; }
>> + make $JOBS || { cat config.log; exit 1; }
>> + cp vswitchd/ovs-vswitchd vswitchd/ovs-vswitchd_with_dpdk
>> + popd
>> + fi
>> +
>> pushd ovs
>> ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}" $* || \
>> - { cat config.log; exit 1; }
>> + { cat config.log; exit 1; }
>> make $JOBS || { cat config.log; exit 1; }
>> popd
>> +
>> + if [ "$DPDK" ]; then
>> + cp ovs/vswitchd/ovs-vswitchd_with_dpdk ovs/vswitchd/ovs-vswitchd
>> + fi
>> }
>>
>> function configure_ovn()
>> @@ -104,6 +145,12 @@ if [ "$TESTSUITE" ]; then
>> execute_system_tests "check-system-userspace" \
>> "system-userspace-testsuite.log"
>> ;;
>> +
>> + "system-test-dpdk")
>> + # The dpdk tests need huge page memory, so reserve some 2M pages.
>> + sudo bash -c "echo 2048 >
>> /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
>> + execute_system_tests "check-system-dpdk"
>> "system-dpdk-testsuite.log"
>> + ;;
>> esac
>> else
>> configure_ovn $OPTS
>> diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
>> index 6e8eac8d1..fe2a14c40 100644
>> --- a/.github/workflows/test.yml
>> +++ b/.github/workflows/test.yml
>> @@ -12,11 +12,81 @@ concurrency:
>> cancel-in-progress: true
>>
>> jobs:
>> + build-dpdk:
>> + env:
>> + dependencies: gcc libnuma-dev ninja-build
>> + CC: gcc
>> + DPDK_GIT: https://dpdk.org/git/dpdk-stable
>> + DPDK_VER: 22.11.1
>> + name: dpdk gcc
>> + outputs:
>> + dpdk_key: ${{ steps.gen_dpdk_key.outputs.key }}
>> + runs-on: ubuntu-20.04
>> + timeout-minutes: 30
>> +
>> + steps:
>> + - name: checkout
>> + uses: actions/checkout@v3
>> +
>> + - name: update PATH
>> + run: |
>> + echo "$HOME/bin" >> $GITHUB_PATH
>> + echo "$HOME/.local/bin" >> $GITHUB_PATH
>> +
>> + - name: create ci signature file for the dpdk cache key
>> + # This will collect most of DPDK related lines, so hash will be
>> different
>> + # if something changed in a way we're building DPDK including
>> DPDK_VER.
>> + # This also allows us to use cache from any branch as long as
>> version
>> + # and a way we're building DPDK stays the same.
>> + run: |
>> + grep -irE 'RTE_|DPDK|meson|ninja' .ci/dpdk-* > dpdk-ci-signature
>> + grep -rwE 'DPDK_GIT|DPDK_VER' .github/ >> dpdk-ci-signature
>> + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then
>> + git ls-remote --heads $DPDK_GIT $DPDK_VER >> dpdk-ci-signature
>> + fi
>> + cat dpdk-ci-signature
>> +
>> + - name: generate ci DPDK key
>> + id: gen_dpdk_key
>> + env:
>> + ci_key: ${{ hashFiles('dpdk-ci-signature') }}
>> + run: echo 'key=dpdk-${{ env.ci_key }}' >> $GITHUB_OUTPUT
>> +
>> + - name: cache
>> + id: dpdk_cache
>> + uses: actions/cache@v3
>> + with:
>> + path: dpdk-dir
>> + key: ${{ steps.gen_dpdk_key.outputs.key }}
>> +
>> + - name: set up python
>> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
>> + uses: actions/setup-python@v4
>> + with:
>> + python-version: '3.9'
>> +
>> + - name: update APT cache
>> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
>> + run: sudo apt update || true
>> + - name: install common dependencies
>> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
>> + run: sudo apt install -y ${{ env.dependencies }}
>> +
>> + - name: prepare
>> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
>> + run: ./.ci/dpdk-prepare.sh
>> +
>> + - name: build
>> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
>> + run: ./.ci/dpdk-build.sh
>> +
>> build-linux:
>> + needs: build-dpdk
>> env:
>> IMAGE_NAME: ghcr.io/ovn-org/ovn-tests:ubuntu
>> ARCH: ${{ matrix.cfg.arch }}
>> CC: ${{ matrix.cfg.compiler }}
>> + DPDK: ${{ matrix.cfg.dpdk }}
>> LIBS: ${{ matrix.cfg.libs }}
>> OPTS: ${{ matrix.cfg.opts }}
>> TESTSUITE: ${{ matrix.cfg.testsuite }}
>> @@ -43,6 +113,9 @@ jobs:
>> - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range:
>> "-500" }
>> - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range:
>> "501-1000" }
>> - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range:
>> "1001-" }
>> + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk,
>> test_range: "-100" }
>> + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk,
>> test_range: "101-200" }
>> + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk,
>> test_range: "201-" }
>> - { compiler: gcc, testsuite: system-test-userspace, test_range:
>> "-100" }
>> - { compiler: gcc, testsuite: system-test-userspace, test_range:
>> "101-200" }
>> - { compiler: gcc, testsuite: system-test-userspace, test_range:
>> "201-" }
>> @@ -84,6 +157,13 @@ jobs:
>> sort -V | tail -1)
>> working-directory: ovs
>>
>> + - name: cache
>> + if: matrix.cfg.dpdk != ''
>> + uses: actions/cache@v3
>> + with:
>> + path: dpdk-dir
>> + key: ${{ needs.build-dpdk.outputs.dpdk_key }}
>> +
>> - name: build
>> if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }}
>> run: sudo -E ./.ci/ci.sh --archive-logs
>> diff --git a/Makefile.am b/Makefile.am
>> index f1bd72d94..27182c7bc 100644
>> --- a/Makefile.am
>> +++ b/Makefile.am
>> @@ -87,6 +87,8 @@ EXTRA_DIST = \
>> NOTICE \
>> .cirrus.yml \
>> .ci/ci.sh \
>> + .ci/dpdk-build.sh \
>> + .ci/dpdk-prepare.sh \
>> .ci/linux-build.sh \
>> .ci/osx-build.sh \
>> .ci/osx-prepare.sh \
>> diff --git a/tests/system-ovn.at b/tests/system-ovn.at
>> index 05c234edc..a3f8fdaa9 100644
>> --- a/tests/system-ovn.at
>> +++ b/tests/system-ovn.at
>> @@ -4267,7 +4267,7 @@ done
>> # Enable IGMP snooping on sw1.
>> ovn-nbctl set Logical_Switch sw1 other_config:mcast_querier="false"
>> ovn-nbctl set Logical_Switch sw1 other_config:mcast_snoop="true"
>> -
>> +ovn-nbctl --wait=hv sync
>>
>> group_v4="239.0.1.68"
>> # Inject IGMP Join for v4 group on sw1-p1.
>>
>> _______________________________________________
>> dev mailing list
>> d...@openvswitch.org
>> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
>>
>>
> Thanks,
> Ales
>
> [0]
> https://github.com/ovn-org/ovn/blob/fd68fd757a8b2c7f1e2828315c3ccde11921b1e3/utilities/containers/py-requirements.txt
>
> --
>
> Ales Musil
>
> Senior Software Engineer - OVN Core
>
> Red Hat EMEA <https://www.redhat.com>
>
> amu...@redhat.com IM: amusil
> <https://red.ht/sig>
_______________________________________________
dev mailing list
d...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
