On Mon, Jun 19, 2023 at 11:32 AM Eelco Chaudron <echau...@redhat.com> wrote:
> This patch includes changes made earlier by David in the > ovs branch to cache the dpdk builds. > > Co-authored-by: David Marchand <david.march...@redhat.com> > Signed-off-by: David Marchand <david.march...@redhat.com> > Signed-off-by: Eelco Chaudron <echau...@redhat.com> > --- > > v2: Replaced 'sleep 1' with '' after consulting with Dumitru. > v3: No changes > > Note that I ran the full GitHub ci 20x on the v1 patchset. > For v2, I did 10 runs of only the changed/fixed test. For v3, > I only did 1 run. > Hi Eelco, I have one comment down below. > > .ci/ci.sh | 12 ++++++- > .ci/dpdk-build.sh | 54 ++++++++++++++++++++++++++++++ > .ci/dpdk-prepare.sh | 11 ++++++ > .ci/linux-build.sh | 49 ++++++++++++++++++++++++++- > .github/workflows/test.yml | 80 > ++++++++++++++++++++++++++++++++++++++++++++ > Makefile.am | 2 + > tests/system-ovn.at | 2 + > 7 files changed, 207 insertions(+), 3 deletions(-) > create mode 100755 .ci/dpdk-build.sh > create mode 100755 .ci/dpdk-prepare.sh > > diff --git a/.ci/ci.sh b/.ci/ci.sh > index 90942bab6..10f11939c 100755 > --- a/.ci/ci.sh > +++ b/.ci/ci.sh > @@ -16,6 +16,7 @@ > > OVN_PATH=${OVN_PATH:-$PWD} > OVS_PATH=${OVS_PATH:-$OVN_PATH/ovs} > +DPDK_PATH=${DPDK_PATH:-$OVN_PATH/dpdk-dir} > CONTAINER_CMD=${CONTAINER_CMD:-podman} > CONTAINER_WORKSPACE="/workspace" > CONTAINER_WORKDIR="/workspace/ovn-tmp" > @@ -80,6 +81,10 @@ function copy_sources_to_workdir() { > && \ > cp -a $CONTAINER_WORKSPACE/ovs/. $CONTAINER_WORKDIR/ovs \ > && \ > + rm -rf $CONTAINER_WORKDIR/dpdk-dir \ > + && \ > + cp -a $CONTAINER_WORKSPACE/dpdk-dir/. $CONTAINER_WORKDIR/dpdk-dir > \ > + && \ > git config --global --add safe.directory $CONTAINER_WORKDIR > " > } > @@ -95,7 +100,7 @@ function run_tests() { > cd $CONTAINER_WORKDIR \ > && \ > ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \ > - TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS \ > + TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \ > ./.ci/linux-build.sh > " > } > @@ -148,12 +153,17 @@ if [ "$ARCH" = "aarch64" ]; then > ASAN_OPTIONS="detect_leaks=0" > fi > > +if [ -z "$DPDK" ]; then > + mkdir -p "$DPDK_PATH" > +fi > + > CONTAINER_ID="$($CONTAINER_CMD run --privileged -d \ > --pids-limit=-1 \ > --env ASAN_OPTIONS=$ASAN_OPTIONS \ > -v /lib/modules/$(uname -r):/lib/modules/$(uname -r):ro \ > -v $OVN_PATH:$CONTAINER_WORKSPACE/ovn:Z \ > -v $OVS_PATH:$CONTAINER_WORKSPACE/ovs:Z \ > + -v $DPDK_PATH:$CONTAINER_WORKSPACE/dpdk-dir:Z \ > $IMAGE_NAME)" > trap remove_container EXIT > > diff --git a/.ci/dpdk-build.sh b/.ci/dpdk-build.sh > new file mode 100755 > index 000000000..f44ac15b0 > --- /dev/null > +++ b/.ci/dpdk-build.sh > @@ -0,0 +1,54 @@ > +#!/bin/bash > + > +set -o errexit > +set -x > + > +function build_dpdk() > +{ > + local VERSION_FILE="dpdk-dir/cached-version" > + local DPDK_VER=$1 > + local DPDK_OPTS="" > + > + rm -rf dpdk-dir > + > + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then > + git clone --single-branch $DPDK_GIT dpdk-dir -b > "${DPDK_VER##refs/*/}" > + pushd dpdk-dir > + git log -1 --oneline > + else > + wget https://fast.dpdk.org/rel/dpdk-$1.tar.xz > + tar xvf dpdk-$1.tar.xz > /dev/null > + DIR_NAME=$(tar -tf dpdk-$1.tar.xz | head -1 | cut -f1 -d"/") > + mv ${DIR_NAME} dpdk-dir > + pushd dpdk-dir > + fi > + > + # Switching to 'default' machine to make dpdk-dir cache usable on > + # different CPUs. We can't be sure that all CI machines are exactly > same. > + DPDK_OPTS="$DPDK_OPTS -Dmachine=default" > + > + # Disable building DPDK unit tests. Not needed for OVS build or tests. > + DPDK_OPTS="$DPDK_OPTS -Dtests=false" > + > + # Disable DPDK developer mode, this results in less build checks and > less > + # meson verbose outputs. > + DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled" > + > + # OVS compilation and the "ovn-system-dpdk" unit tests (run in the CI) > + # only depend on virtio/tap drivers. > + # We can disable all remaining drivers to save compilation time. > + DPDK_OPTS="$DPDK_OPTS -Denable_drivers=net/null,net/tap,net/virtio" > + > + # Install DPDK using prefix. > + DPDK_OPTS="$DPDK_OPTS --prefix=$(pwd)/build" > + > + meson $DPDK_OPTS build > + ninja -C build > + ninja -C build install > + > + echo "Installed DPDK in $(pwd)" > + popd > + echo "${DPDK_VER}" > ${VERSION_FILE} > +} > + > +build_dpdk $DPDK_VER > diff --git a/.ci/dpdk-prepare.sh b/.ci/dpdk-prepare.sh > new file mode 100755 > index 000000000..f7e6215dd > --- /dev/null > +++ b/.ci/dpdk-prepare.sh > @@ -0,0 +1,11 @@ > +#!/bin/bash > + > +set -ev > + > +# Installing wheel separately because it may be needed to build some > +# of the packages during dependency backtracking and pip >= 22.0 will > +# abort backtracking on build failures: > +# https://github.com/pypa/pip/issues/10655 > +pip3 install --disable-pip-version-check --user wheel > +pip3 install --disable-pip-version-check --user pyelftools > +pip3 install --user 'meson==0.53.2' > Do we need this whole dpdk-prepare.sh? We have moved all python dependencies to py-requirements.txt [0]. pyelftools are already there so the only missing package would be meson. > diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh > index 907a0dc6c..5a79a52da 100755 > --- a/.ci/linux-build.sh > +++ b/.ci/linux-build.sh > @@ -10,13 +10,54 @@ OVN_CFLAGS="" > OPTS="$OPTS --enable-Werror" > JOBS=${JOBS:-"-j4"} > > +function install_dpdk() > +{ > + local VERSION_FILE="dpdk-dir/cached-version" > + local DPDK_LIB=$(pwd)/dpdk-dir/build/lib/x86_64-linux-gnu > + > + # Export the following path for pkg-config to find the .pc file. > + export PKG_CONFIG_PATH=$DPDK_LIB/pkgconfig/:$PKG_CONFIG_PATH > + > + if [ ! -f "${VERSION_FILE}" ]; then > + echo "Could not find DPDK in $(pwd)/dpdk-dir" > + return 1 > + fi > + > + # As we build inside a container we need to update the prefix. > + sed -i -E "s|^prefix=.*|prefix=$(pwd)/dpdk-dir/build|" \ > + "$DPDK_LIB/pkgconfig/libdpdk-libs.pc" > + > + # Update the library paths. > + sudo ldconfig > + echo "Found cached DPDK $(cat ${VERSION_FILE}) build in > $(pwd)/dpdk-dir" > +} > + > function configure_ovs() > { > + if [ "$DPDK" ]; then > + # When DPDK is enabled, we need to build OVS twice. Once to have > + # ovs-vswitchd with DPDK. But OVN does not like the OVS libraries > to > + # be compiled with DPDK enabled, hence we need a final clean build > + # with this disabled. > + install_dpdk > + > + pushd ovs > + ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}" > --with-dpdk=static \ > + $* || { cat config.log; exit 1; } > + make $JOBS || { cat config.log; exit 1; } > + cp vswitchd/ovs-vswitchd vswitchd/ovs-vswitchd_with_dpdk > + popd > + fi > + > pushd ovs > ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}" $* || \ > - { cat config.log; exit 1; } > + { cat config.log; exit 1; } > make $JOBS || { cat config.log; exit 1; } > popd > + > + if [ "$DPDK" ]; then > + cp ovs/vswitchd/ovs-vswitchd_with_dpdk ovs/vswitchd/ovs-vswitchd > + fi > } > > function configure_ovn() > @@ -104,6 +145,12 @@ if [ "$TESTSUITE" ]; then > execute_system_tests "check-system-userspace" \ > "system-userspace-testsuite.log" > ;; > + > + "system-test-dpdk") > + # The dpdk tests need huge page memory, so reserve some 2M pages. > + sudo bash -c "echo 2048 > > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" > + execute_system_tests "check-system-dpdk" > "system-dpdk-testsuite.log" > + ;; > esac > else > configure_ovn $OPTS > diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml > index 6e8eac8d1..fe2a14c40 100644 > --- a/.github/workflows/test.yml > +++ b/.github/workflows/test.yml > @@ -12,11 +12,81 @@ concurrency: > cancel-in-progress: true > > jobs: > + build-dpdk: > + env: > + dependencies: gcc libnuma-dev ninja-build > + CC: gcc > + DPDK_GIT: https://dpdk.org/git/dpdk-stable > + DPDK_VER: 22.11.1 > + name: dpdk gcc > + outputs: > + dpdk_key: ${{ steps.gen_dpdk_key.outputs.key }} > + runs-on: ubuntu-20.04 > + timeout-minutes: 30 > + > + steps: > + - name: checkout > + uses: actions/checkout@v3 > + > + - name: update PATH > + run: | > + echo "$HOME/bin" >> $GITHUB_PATH > + echo "$HOME/.local/bin" >> $GITHUB_PATH > + > + - name: create ci signature file for the dpdk cache key > + # This will collect most of DPDK related lines, so hash will be > different > + # if something changed in a way we're building DPDK including > DPDK_VER. > + # This also allows us to use cache from any branch as long as > version > + # and a way we're building DPDK stays the same. > + run: | > + grep -irE 'RTE_|DPDK|meson|ninja' .ci/dpdk-* > dpdk-ci-signature > + grep -rwE 'DPDK_GIT|DPDK_VER' .github/ >> dpdk-ci-signature > + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then > + git ls-remote --heads $DPDK_GIT $DPDK_VER >> dpdk-ci-signature > + fi > + cat dpdk-ci-signature > + > + - name: generate ci DPDK key > + id: gen_dpdk_key > + env: > + ci_key: ${{ hashFiles('dpdk-ci-signature') }} > + run: echo 'key=dpdk-${{ env.ci_key }}' >> $GITHUB_OUTPUT > + > + - name: cache > + id: dpdk_cache > + uses: actions/cache@v3 > + with: > + path: dpdk-dir > + key: ${{ steps.gen_dpdk_key.outputs.key }} > + > + - name: set up python > + if: steps.dpdk_cache.outputs.cache-hit != 'true' > + uses: actions/setup-python@v4 > + with: > + python-version: '3.9' > + > + - name: update APT cache > + if: steps.dpdk_cache.outputs.cache-hit != 'true' > + run: sudo apt update || true > + - name: install common dependencies > + if: steps.dpdk_cache.outputs.cache-hit != 'true' > + run: sudo apt install -y ${{ env.dependencies }} > + > + - name: prepare > + if: steps.dpdk_cache.outputs.cache-hit != 'true' > + run: ./.ci/dpdk-prepare.sh > + > + - name: build > + if: steps.dpdk_cache.outputs.cache-hit != 'true' > + run: ./.ci/dpdk-build.sh > + > build-linux: > + needs: build-dpdk > env: > IMAGE_NAME: ghcr.io/ovn-org/ovn-tests:ubuntu > ARCH: ${{ matrix.cfg.arch }} > CC: ${{ matrix.cfg.compiler }} > + DPDK: ${{ matrix.cfg.dpdk }} > LIBS: ${{ matrix.cfg.libs }} > OPTS: ${{ matrix.cfg.opts }} > TESTSUITE: ${{ matrix.cfg.testsuite }} > @@ -43,6 +113,9 @@ jobs: > - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: > "-500" } > - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: > "501-1000" } > - { compiler: gcc, testsuite: test, libs: -ljemalloc, test_range: > "1001-" } > + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, > test_range: "-100" } > + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, > test_range: "101-200" } > + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk, > test_range: "201-" } > - { compiler: gcc, testsuite: system-test-userspace, test_range: > "-100" } > - { compiler: gcc, testsuite: system-test-userspace, test_range: > "101-200" } > - { compiler: gcc, testsuite: system-test-userspace, test_range: > "201-" } > @@ -84,6 +157,13 @@ jobs: > sort -V | tail -1) > working-directory: ovs > > + - name: cache > + if: matrix.cfg.dpdk != '' > + uses: actions/cache@v3 > + with: > + path: dpdk-dir > + key: ${{ needs.build-dpdk.outputs.dpdk_key }} > + > - name: build > if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }} > run: sudo -E ./.ci/ci.sh --archive-logs > diff --git a/Makefile.am b/Makefile.am > index f1bd72d94..27182c7bc 100644 > --- a/Makefile.am > +++ b/Makefile.am > @@ -87,6 +87,8 @@ EXTRA_DIST = \ > NOTICE \ > .cirrus.yml \ > .ci/ci.sh \ > + .ci/dpdk-build.sh \ > + .ci/dpdk-prepare.sh \ > .ci/linux-build.sh \ > .ci/osx-build.sh \ > .ci/osx-prepare.sh \ > diff --git a/tests/system-ovn.at b/tests/system-ovn.at > index 05c234edc..a3f8fdaa9 100644 > --- a/tests/system-ovn.at > +++ b/tests/system-ovn.at > @@ -4267,7 +4267,7 @@ done > # Enable IGMP snooping on sw1. > ovn-nbctl set Logical_Switch sw1 other_config:mcast_querier="false" > ovn-nbctl set Logical_Switch sw1 other_config:mcast_snoop="true" > - > +ovn-nbctl --wait=hv sync > > group_v4="239.0.1.68" > # Inject IGMP Join for v4 group on sw1-p1. > > _______________________________________________ > dev mailing list > d...@openvswitch.org > https://mail.openvswitch.org/mailman/listinfo/ovs-dev > > Thanks, Ales [0] https://github.com/ovn-org/ovn/blob/fd68fd757a8b2c7f1e2828315c3ccde11921b1e3/utilities/containers/py-requirements.txt -- Ales Musil Senior Software Engineer - OVN Core Red Hat EMEA <https://www.redhat.com> amu...@redhat.com IM: amusil <https://red.ht/sig> _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev