On Tue, Jul 4, 2023 at 12:53 PM Eelco Chaudron <echau...@redhat.com> wrote:
>
>
> On 4 Jul 2023, at 11:00, Ales Musil wrote:
>
> > On Mon, Jun 19, 2023 at 11:32 AM Eelco Chaudron <echau...@redhat.com>
> wrote:
> >
> >> This patch includes changes made earlier by David in the
> >> ovs branch to cache the dpdk builds.
> >>
> >> Co-authored-by: David Marchand <david.march...@redhat.com>
> >> Signed-off-by: David Marchand <david.march...@redhat.com>
> >> Signed-off-by: Eelco Chaudron <echau...@redhat.com>
> >> ---
> >>
> >> v2: Replaced 'sleep 1' with '' after consulting with Dumitru.
> >> v3: No changes
> >>
> >> Note that I ran the full GitHub ci 20x on the v1 patchset.
> >> For v2, I did 10 runs of only the changed/fixed test. For v3,
> >> I only did 1 run.
> >>
> >
> > Hi Eelco,
> >
> > I have one comment down below.
> >
> >
> >>
> >> .ci/ci.sh | 12 ++++++-
> >> .ci/dpdk-build.sh | 54 ++++++++++++++++++++++++++++++
> >> .ci/dpdk-prepare.sh | 11 ++++++
> >> .ci/linux-build.sh | 49 ++++++++++++++++++++++++++-
> >> .github/workflows/test.yml | 80
> >> ++++++++++++++++++++++++++++++++++++++++++++
> >> Makefile.am | 2 +
> >> tests/system-ovn.at | 2 +
> >> 7 files changed, 207 insertions(+), 3 deletions(-)
> >> create mode 100755 .ci/dpdk-build.sh
> >> create mode 100755 .ci/dpdk-prepare.sh
> >>
> >> diff --git a/.ci/ci.sh b/.ci/ci.sh
> >> index 90942bab6..10f11939c 100755
> >> --- a/.ci/ci.sh
> >> +++ b/.ci/ci.sh
> >> @@ -16,6 +16,7 @@
> >>
> >> OVN_PATH=${OVN_PATH:-$PWD}
> >> OVS_PATH=${OVS_PATH:-$OVN_PATH/ovs}
> >> +DPDK_PATH=${DPDK_PATH:-$OVN_PATH/dpdk-dir}
> >> CONTAINER_CMD=${CONTAINER_CMD:-podman}
> >> CONTAINER_WORKSPACE="/workspace"
> >> CONTAINER_WORKDIR="/workspace/ovn-tmp"
> >> @@ -80,6 +81,10 @@ function copy_sources_to_workdir() {
> >> && \
> >> cp -a $CONTAINER_WORKSPACE/ovs/. $CONTAINER_WORKDIR/ovs \
> >> && \
> >> + rm -rf $CONTAINER_WORKDIR/dpdk-dir \
> >> + && \
> >> + cp -a $CONTAINER_WORKSPACE/dpdk-dir/.
> $CONTAINER_WORKDIR/dpdk-dir
> >> \
> >> + && \
> >> git config --global --add safe.directory $CONTAINER_WORKDIR
> >> "
> >> }
> >> @@ -95,7 +100,7 @@ function run_tests() {
> >> cd $CONTAINER_WORKDIR \
> >> && \
> >> ARCH=$ARCH CC=$CC LIBS=$LIBS OPTS=$OPTS TESTSUITE=$TESTSUITE \
> >> - TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS \
> >> + TEST_RANGE=$TEST_RANGE SANITIZERS=$SANITIZERS DPDK=$DPDK \
> >> ./.ci/linux-build.sh
> >> "
> >> }
> >> @@ -148,12 +153,17 @@ if [ "$ARCH" = "aarch64" ]; then
> >> ASAN_OPTIONS="detect_leaks=0"
> >> fi
> >>
> >> +if [ -z "$DPDK" ]; then
> >> + mkdir -p "$DPDK_PATH"
> >> +fi
> >> +
> >> CONTAINER_ID="$($CONTAINER_CMD run --privileged -d \
> >> --pids-limit=-1 \
> >> --env ASAN_OPTIONS=$ASAN_OPTIONS \
> >> -v /lib/modules/$(uname -r):/lib/modules/$(uname -r):ro \
> >> -v $OVN_PATH:$CONTAINER_WORKSPACE/ovn:Z \
> >> -v $OVS_PATH:$CONTAINER_WORKSPACE/ovs:Z \
> >> + -v $DPDK_PATH:$CONTAINER_WORKSPACE/dpdk-dir:Z \
> >> $IMAGE_NAME)"
> >> trap remove_container EXIT
> >>
> >> diff --git a/.ci/dpdk-build.sh b/.ci/dpdk-build.sh
> >> new file mode 100755
> >> index 000000000..f44ac15b0
> >> --- /dev/null
> >> +++ b/.ci/dpdk-build.sh
> >> @@ -0,0 +1,54 @@
> >> +#!/bin/bash
> >> +
> >> +set -o errexit
> >> +set -x
> >> +
> >> +function build_dpdk()
> >> +{
> >> + local VERSION_FILE="dpdk-dir/cached-version"
> >> + local DPDK_VER=$1
> >> + local DPDK_OPTS=""
> >> +
> >> + rm -rf dpdk-dir
> >> +
> >> + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then
> >> + git clone --single-branch $DPDK_GIT dpdk-dir -b
> >> "${DPDK_VER##refs/*/}"
> >> + pushd dpdk-dir
> >> + git log -1 --oneline
> >> + else
> >> + wget https://fast.dpdk.org/rel/dpdk-$1.tar.xz
> >> + tar xvf dpdk-$1.tar.xz > /dev/null
> >> + DIR_NAME=$(tar -tf dpdk-$1.tar.xz | head -1 | cut -f1 -d"/")
> >> + mv ${DIR_NAME} dpdk-dir
> >> + pushd dpdk-dir
> >> + fi
> >> +
> >> + # Switching to 'default' machine to make dpdk-dir cache usable on
> >> + # different CPUs. We can't be sure that all CI machines are exactly
> >> same.
> >> + DPDK_OPTS="$DPDK_OPTS -Dmachine=default"
> >> +
> >> + # Disable building DPDK unit tests. Not needed for OVS build or
> tests.
> >> + DPDK_OPTS="$DPDK_OPTS -Dtests=false"
> >> +
> >> + # Disable DPDK developer mode, this results in less build checks
> and
> >> less
> >> + # meson verbose outputs.
> >> + DPDK_OPTS="$DPDK_OPTS -Ddeveloper_mode=disabled"
> >> +
> >> + # OVS compilation and the "ovn-system-dpdk" unit tests (run in the
> CI)
> >> + # only depend on virtio/tap drivers.
> >> + # We can disable all remaining drivers to save compilation time.
> >> + DPDK_OPTS="$DPDK_OPTS -Denable_drivers=net/null,net/tap,net/virtio"
> >> +
> >> + # Install DPDK using prefix.
> >> + DPDK_OPTS="$DPDK_OPTS --prefix=$(pwd)/build"
> >> +
> >> + meson $DPDK_OPTS build
> >> + ninja -C build
> >> + ninja -C build install
> >> +
> >> + echo "Installed DPDK in $(pwd)"
> >> + popd
> >> + echo "${DPDK_VER}" > ${VERSION_FILE}
> >> +}
> >> +
> >> +build_dpdk $DPDK_VER
> >> diff --git a/.ci/dpdk-prepare.sh b/.ci/dpdk-prepare.sh
> >> new file mode 100755
> >> index 000000000..f7e6215dd
> >> --- /dev/null
> >> +++ b/.ci/dpdk-prepare.sh
> >> @@ -0,0 +1,11 @@
> >> +#!/bin/bash
> >> +
> >> +set -ev
> >> +
> >> +# Installing wheel separately because it may be needed to build some
> >> +# of the packages during dependency backtracking and pip >= 22.0 will
> >> +# abort backtracking on build failures:
> >> +# https://github.com/pypa/pip/issues/10655
> >> +pip3 install --disable-pip-version-check --user wheel
> >> +pip3 install --disable-pip-version-check --user pyelftools
> >> +pip3 install --user 'meson==0.53.2'
> >>
> >
> > Do we need this whole dpdk-prepare.sh? We have moved all python
> > dependencies to
> > py-requirements.txt [0]. pyelftools are already there so the only missing
> > package would be meson.
>
> The dpdk cash is built outside of the containers, so we need this.
>
Hmm that's not very nice, but I don't have a better idea.
Acked-by: Ales Musil <amu...@redhat.com>
Thanks,
Ales
>
> >> diff --git a/.ci/linux-build.sh b/.ci/linux-build.sh
> >> index 907a0dc6c..5a79a52da 100755
> >> --- a/.ci/linux-build.sh
> >> +++ b/.ci/linux-build.sh
> >> @@ -10,13 +10,54 @@ OVN_CFLAGS=""
> >> OPTS="$OPTS --enable-Werror"
> >> JOBS=${JOBS:-"-j4"}
> >>
> >> +function install_dpdk()
> >> +{
> >> + local VERSION_FILE="dpdk-dir/cached-version"
> >> + local DPDK_LIB=$(pwd)/dpdk-dir/build/lib/x86_64-linux-gnu
> >> +
> >> + # Export the following path for pkg-config to find the .pc file.
> >> + export PKG_CONFIG_PATH=$DPDK_LIB/pkgconfig/:$PKG_CONFIG_PATH
> >> +
> >> + if [ ! -f "${VERSION_FILE}" ]; then
> >> + echo "Could not find DPDK in $(pwd)/dpdk-dir"
> >> + return 1
> >> + fi
> >> +
> >> + # As we build inside a container we need to update the prefix.
> >> + sed -i -E "s|^prefix=.*|prefix=$(pwd)/dpdk-dir/build|" \
> >> + "$DPDK_LIB/pkgconfig/libdpdk-libs.pc"
> >> +
> >> + # Update the library paths.
> >> + sudo ldconfig
> >> + echo "Found cached DPDK $(cat ${VERSION_FILE}) build in
> >> $(pwd)/dpdk-dir"
> >> +}
> >> +
> >> function configure_ovs()
> >> {
> >> + if [ "$DPDK" ]; then
> >> + # When DPDK is enabled, we need to build OVS twice. Once to
> have
> >> + # ovs-vswitchd with DPDK. But OVN does not like the OVS
> libraries
> >> to
> >> + # be compiled with DPDK enabled, hence we need a final clean
> build
> >> + # with this disabled.
> >> + install_dpdk
> >> +
> >> + pushd ovs
> >> + ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}"
> >> --with-dpdk=static \
> >> + $* || { cat config.log; exit 1; }
> >> + make $JOBS || { cat config.log; exit 1; }
> >> + cp vswitchd/ovs-vswitchd vswitchd/ovs-vswitchd_with_dpdk
> >> + popd
> >> + fi
> >> +
> >> pushd ovs
> >> ./boot.sh && ./configure CFLAGS="${COMMON_CFLAGS}" $* || \
> >> - { cat config.log; exit 1; }
> >> + { cat config.log; exit 1; }
> >> make $JOBS || { cat config.log; exit 1; }
> >> popd
> >> +
> >> + if [ "$DPDK" ]; then
> >> + cp ovs/vswitchd/ovs-vswitchd_with_dpdk
> ovs/vswitchd/ovs-vswitchd
> >> + fi
> >> }
> >>
> >> function configure_ovn()
> >> @@ -104,6 +145,12 @@ if [ "$TESTSUITE" ]; then
> >> execute_system_tests "check-system-userspace" \
> >> "system-userspace-testsuite.log"
> >> ;;
> >> +
> >> + "system-test-dpdk")
> >> + # The dpdk tests need huge page memory, so reserve some 2M
> pages.
> >> + sudo bash -c "echo 2048 >
> >> /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages"
> >> + execute_system_tests "check-system-dpdk"
> >> "system-dpdk-testsuite.log"
> >> + ;;
> >> esac
> >> else
> >> configure_ovn $OPTS
> >> diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
> >> index 6e8eac8d1..fe2a14c40 100644
> >> --- a/.github/workflows/test.yml
> >> +++ b/.github/workflows/test.yml
> >> @@ -12,11 +12,81 @@ concurrency:
> >> cancel-in-progress: true
> >>
> >> jobs:
> >> + build-dpdk:
> >> + env:
> >> + dependencies: gcc libnuma-dev ninja-build
> >> + CC: gcc
> >> + DPDK_GIT: https://dpdk.org/git/dpdk-stable
> >> + DPDK_VER: 22.11.1
> >> + name: dpdk gcc
> >> + outputs:
> >> + dpdk_key: ${{ steps.gen_dpdk_key.outputs.key }}
> >> + runs-on: ubuntu-20.04
> >> + timeout-minutes: 30
> >> +
> >> + steps:
> >> + - name: checkout
> >> + uses: actions/checkout@v3
> >> +
> >> + - name: update PATH
> >> + run: |
> >> + echo "$HOME/bin" >> $GITHUB_PATH
> >> + echo "$HOME/.local/bin" >> $GITHUB_PATH
> >> +
> >> + - name: create ci signature file for the dpdk cache key
> >> + # This will collect most of DPDK related lines, so hash will be
> >> different
> >> + # if something changed in a way we're building DPDK including
> >> DPDK_VER.
> >> + # This also allows us to use cache from any branch as long as
> >> version
> >> + # and a way we're building DPDK stays the same.
> >> + run: |
> >> + grep -irE 'RTE_|DPDK|meson|ninja' .ci/dpdk-* >
> dpdk-ci-signature
> >> + grep -rwE 'DPDK_GIT|DPDK_VER' .github/ >> dpdk-ci-signature
> >> + if [ "${DPDK_VER##refs/*/}" != "${DPDK_VER}" ]; then
> >> + git ls-remote --heads $DPDK_GIT $DPDK_VER >>
> dpdk-ci-signature
> >> + fi
> >> + cat dpdk-ci-signature
> >> +
> >> + - name: generate ci DPDK key
> >> + id: gen_dpdk_key
> >> + env:
> >> + ci_key: ${{ hashFiles('dpdk-ci-signature') }}
> >> + run: echo 'key=dpdk-${{ env.ci_key }}' >> $GITHUB_OUTPUT
> >> +
> >> + - name: cache
> >> + id: dpdk_cache
> >> + uses: actions/cache@v3
> >> + with:
> >> + path: dpdk-dir
> >> + key: ${{ steps.gen_dpdk_key.outputs.key }}
> >> +
> >> + - name: set up python
> >> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
> >> + uses: actions/setup-python@v4
> >> + with:
> >> + python-version: '3.9'
> >> +
> >> + - name: update APT cache
> >> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
> >> + run: sudo apt update || true
> >> + - name: install common dependencies
> >> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
> >> + run: sudo apt install -y ${{ env.dependencies }}
> >> +
> >> + - name: prepare
> >> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
> >> + run: ./.ci/dpdk-prepare.sh
> >> +
> >> + - name: build
> >> + if: steps.dpdk_cache.outputs.cache-hit != 'true'
> >> + run: ./.ci/dpdk-build.sh
> >> +
> >> build-linux:
> >> + needs: build-dpdk
> >> env:
> >> IMAGE_NAME: ghcr.io/ovn-org/ovn-tests:ubuntu
> >> ARCH: ${{ matrix.cfg.arch }}
> >> CC: ${{ matrix.cfg.compiler }}
> >> + DPDK: ${{ matrix.cfg.dpdk }}
> >> LIBS: ${{ matrix.cfg.libs }}
> >> OPTS: ${{ matrix.cfg.opts }}
> >> TESTSUITE: ${{ matrix.cfg.testsuite }}
> >> @@ -43,6 +113,9 @@ jobs:
> >> - { compiler: gcc, testsuite: test, libs: -ljemalloc,
> test_range:
> >> "-500" }
> >> - { compiler: gcc, testsuite: test, libs: -ljemalloc,
> test_range:
> >> "501-1000" }
> >> - { compiler: gcc, testsuite: test, libs: -ljemalloc,
> test_range:
> >> "1001-" }
> >> + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk,
> >> test_range: "-100" }
> >> + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk,
> >> test_range: "101-200" }
> >> + - { compiler: gcc, testsuite: system-test-dpdk, dpdk: dpdk,
> >> test_range: "201-" }
> >> - { compiler: gcc, testsuite: system-test-userspace,
> test_range:
> >> "-100" }
> >> - { compiler: gcc, testsuite: system-test-userspace,
> test_range:
> >> "101-200" }
> >> - { compiler: gcc, testsuite: system-test-userspace,
> test_range:
> >> "201-" }
> >> @@ -84,6 +157,13 @@ jobs:
> >> sort -V | tail -1)
> >> working-directory: ovs
> >>
> >> + - name: cache
> >> + if: matrix.cfg.dpdk != ''
> >> + uses: actions/cache@v3
> >> + with:
> >> + path: dpdk-dir
> >> + key: ${{ needs.build-dpdk.outputs.dpdk_key }}
> >> +
> >> - name: build
> >> if: ${{ startsWith(matrix.cfg.testsuite, 'system-test') }}
> >> run: sudo -E ./.ci/ci.sh --archive-logs
> >> diff --git a/Makefile.am b/Makefile.am
> >> index f1bd72d94..27182c7bc 100644
> >> --- a/Makefile.am
> >> +++ b/Makefile.am
> >> @@ -87,6 +87,8 @@ EXTRA_DIST = \
> >> NOTICE \
> >> .cirrus.yml \
> >> .ci/ci.sh \
> >> + .ci/dpdk-build.sh \
> >> + .ci/dpdk-prepare.sh \
> >> .ci/linux-build.sh \
> >> .ci/osx-build.sh \
> >> .ci/osx-prepare.sh \
> >> diff --git a/tests/system-ovn.at b/tests/system-ovn.at
> >> index 05c234edc..a3f8fdaa9 100644
> >> --- a/tests/system-ovn.at
> >> +++ b/tests/system-ovn.at
> >> @@ -4267,7 +4267,7 @@ done
> >> # Enable IGMP snooping on sw1.
> >> ovn-nbctl set Logical_Switch sw1 other_config:mcast_querier="false"
> >> ovn-nbctl set Logical_Switch sw1 other_config:mcast_snoop="true"
> >> -
> >> +ovn-nbctl --wait=hv sync
> >>
> >> group_v4="239.0.1.68"
> >> # Inject IGMP Join for v4 group on sw1-p1.
> >>
> >> _______________________________________________
> >> dev mailing list
> >> d...@openvswitch.org
> >> https://mail.openvswitch.org/mailman/listinfo/ovs-dev
> >>
> >>
> > Thanks,
> > Ales
> >
> > [0]
> >
> https://github.com/ovn-org/ovn/blob/fd68fd757a8b2c7f1e2828315c3ccde11921b1e3/utilities/containers/py-requirements.txt
> >
> > --
> >
> > Ales Musil
> >
> > Senior Software Engineer - OVN Core
> >
> > Red Hat EMEA <https://www.redhat.com>
> >
> > amu...@redhat.com IM: amusil
> > <https://red.ht/sig>
>
>
--
Ales Musil
Senior Software Engineer - OVN Core
Red Hat EMEA <https://www.redhat.com>
amu...@redhat.com IM: amusil
<https://red.ht/sig>
_______________________________________________
dev mailing list
d...@openvswitch.org
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
