This is an automated email from the ASF dual-hosted git repository.
bneradt pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/trafficserver-ci.git
The following commit(s) were added to refs/heads/main by this push:
new 68eff52 Starting fedora:39 images (#203)
68eff52 is described below
commit 68eff52c24d4c27aca69e60b784589547fc5dc17
Author: Brian Neradt <brian.ner...@gmail.com>
AuthorDate: Wed Aug 16 18:15:29 2023 -0500
Starting fedora:39 images (#203)
fedora:39 is in pre-beta so far, but might as well start the Dockerfile,
create an early image, try it out, and update as time goes on. We
probably won't use this in CI yet until we get closer to the release
date in a couple months.
It's early enough that this Dockerfile fails with even a dnf update.
Eventually the upstream fedora:39 should be stable enough for us to
create images to test with.
---
docker/fedora39/Dockerfile | 118 ++++++++++++++++
docker/fedora39/build_h3_tools.sh | 280 ++++++++++++++++++++++++++++++++++++++
2 files changed, 398 insertions(+)
diff --git a/docker/fedora39/Dockerfile b/docker/fedora39/Dockerfile
new file mode 100644
index 0000000..5a529ed
--- /dev/null
+++ b/docker/fedora39/Dockerfile
@@ -0,0 +1,118 @@
+FROM fedora:39
+
+#-------------------------------------------------------------------------------
+# Install the various system packages we use.
+#-------------------------------------------------------------------------------
+RUN <<EOF
+ set -e
+
+ dnf -y install dnf-plugins-core
+ dnf repolist
+ dnf -y update
+
+ # Build tools.
+ dnf -y install \
+ ccache make pkgconfig bison flex gcc-c++ clang \
+ autoconf automake libtool \
+ cmake ninja-build
+
+ # Various other tools
+ dnf -y install \
+ sudo git rpm-build distcc-server file wget openssl hwloc \
+ nghttp2 libnghttp2-devel
+
+ # Devel packages that ATS needs
+ dnf -y install \
+ openssl-devel expat-devel pcre-devel libcap-devel hwloc-devel
libunwind-devel \
+ xz-devel libcurl-devel ncurses-devel jemalloc-devel GeoIP-devel
luajit-devel brotli-devel \
+ ImageMagick-devel ImageMagick-c++-devel hiredis-devel zlib-devel
libmaxminddb-devel \
+ perl-ExtUtils-MakeMaker perl-Digest-SHA perl-URI perl-IPC-Cmd
perl-Pod-Html \
+ curl tcl-devel java
+
+ # autest stuff
+ dnf -y install \
+ python3 httpd-tools procps-ng nmap-ncat python3-pip \
+ python3-gunicorn python3-requests python3-devel python3-psutil telnet
+EOF
+
+#-------------------------------------------------------------------------------
+# Install some custom build tools.
+#-------------------------------------------------------------------------------
+
+WORKDIR /root
+
+# We put our custom packages in /opt.
+RUN <<EOF
+ set -e
+ mkdir -p /opt/bin
+ chmod 755 /opt/bin
+ echo 'PATH=/opt/bin:$PATH' | tee -a /etc/profile.d/opt_bin.sh
+EOF
+ARG PATH=/opt/bin:$PATH
+
+# This currently fails with latest fedora:38. Presumably the stock pip is
+# recent enough, so this shouldn't be a big deal.
+# RUN pip3 install --upgrade pip
+RUN pip3 install pipenv httpbin
+
+#-------------------------------------------------------------------------------
+# Install the HTTP/3 build tools, including openssl-quic.
+#-------------------------------------------------------------------------------
+RUN dnf -y install libev-devel jemalloc-devel libxml2-devel \
+ c-ares-devel libevent-devel jansson-devel zlib-devel systemd-devel \
+ perl-FindBin cargo
+
+# go will be installed by build_h3_tools.
+RUN dnf remove -y golang
+ARG h3_tools_dir=/root/build_h3_tools
+RUN mkdir -p ${h3_tools_dir}
+COPY /build_h3_tools.sh ${h3_tools_dir}/build_h3_tools.sh
+# This will install OpenSSL QUIC and related tools in /opt.
+RUN <<EOF
+ set -e
+ cd ${h3_tools_dir}
+ bash ${h3_tools_dir}/build_h3_tools.sh; \
+ rm -rf ${h3_tools_dir} /root/.rustup
+EOF
+
+
+#-------------------------------------------------------------------------------
+# Various CI Job and Test Requirements.
+#-------------------------------------------------------------------------------
+
+# Autests require some go applications.
+RUN <<EOF
+ set -e
+ echo 'export PATH=$PATH:/opt/go/bin' | tee -a /etc/profile.d/go.sh
+ echo 'export GOBIN=/opt/go/bin' | tee -a /etc/profile.d/go.sh
+
+ /opt/go/bin/go install github.com/summerwind/h2spec/cmd/h2spec@latest
+ cp /root/go/bin/h2spec /opt/go/bin/
+
+ /opt/go/bin/go install
github.com/mccutchen/go-httpbin/v2/cmd/go-httpbin@v2.6.0
+ cp /root/go/bin/go-httpbin /opt/go/bin/
+EOF
+
+# Add the CI's test user. N.B: 1200 is the uid that our jenkins user is
+# configured with, so that has to be used. Otherwise there will be permissions
+# issues.
+ARG username=jenkins
+ARG uid=1200
+RUN <<EOF
+ set -e
+ useradd \
+ --home-dir /home/${username} \
+ --groups users,wheel \
+ --uid ${uid} \
+ --shell /bin/bash \
+ --create-home \
+ ${username}
+ echo "${username} ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
+ chown -R ${username} /home/${username}
+EOF
+
+# Install lcov
+RUN dnf install -y lcov
+
+# Keep this at the end to clean up the dnf cache.
+RUN dnf clean all
diff --git a/docker/fedora39/build_h3_tools.sh
b/docker/fedora39/build_h3_tools.sh
new file mode 100644
index 0000000..9ae66f3
--- /dev/null
+++ b/docker/fedora39/build_h3_tools.sh
@@ -0,0 +1,280 @@
+#!/usr/bin/env bash
+#
+# Simple script to build OpenSSL and various tools with H3 and QUIC support.
+# This probably needs to be modified based on platform.
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+
+# This is a slightly modified version of:
+#
https://github.com/apache/trafficserver/blob/19dfdd4753232d0b77ca555f7ef5f5ba3d2ccae1/tools/build_h3_tools.sh
+#
+# This present script been modified from the latter in the following ways:
+#
+# * This version checks out specific commits of the repos so that people
+# creating images from the corresponding Dockerfile do not get different
+# versions of these over time.
+#
+# * It also doesn't run sudo since the Dockerfile will run this as root.
+#
+# * It also doesn't use a mktemp since the caller sets up a temporary directory
+# that it later removes.
+
+# Update this as the draft we support updates.
+OPENSSL_BRANCH=${OPENSSL_BRANCH:-"openssl-3.1.0+quic+locks"}
+
+# Set these, if desired, to change these to your preferred installation
+# directory
+BASE=${BASE:-"/opt"}
+OPENSSL_BASE=${OPENSSL_BASE:-"${BASE}/openssl-quic"}
+OPENSSL_PREFIX=${OPENSSL_PREFIX:-"${OPENSSL_BASE}-${OPENSSL_BRANCH}"}
+MAKE="make"
+
+CFLAGS=${CFLAGS:-"-O3 -g"}
+CXXFLAGS=${CXXFLAGS:-"-O3 -g"}
+
+if [ -e /etc/redhat-release ]; then
+ MAKE="gmake"
+ TMP_QUICHE_BSSL_PATH="${BASE}/boringssl/lib64"
+ echo
"+-------------------------------------------------------------------------+"
+ echo "| You probably need to run this, or something like this, for your
system: |"
+ echo "|
|"
+ echo "| sudo yum -y install libev-devel jemalloc-devel python2-devel
|"
+ echo "| sudo yum -y install libxml2-devel c-ares-devel libevent-devel
|"
+ echo "| sudo yum -y install jansson-devel zlib-devel systemd-devel cargo
|"
+ echo "|
|"
+ echo "| Rust may be needed too, see https://rustup.rs for the details
|"
+ echo
"+-------------------------------------------------------------------------+"
+ echo
+ echo
+elif [ -e /etc/debian_version ]; then
+ TMP_QUICHE_BSSL_PATH="${BASE}/boringssl/lib"
+ echo
"+-------------------------------------------------------------------------+"
+ echo "| You probably need to run this, or something like this, for your
system: |"
+ echo "|
|"
+ echo "| sudo apt -y install libev-dev libjemalloc-dev python2-dev
libxml2-dev |"
+ echo "| sudo apt -y install libpython2-dev libc-ares-dev libsystemd-dev
|"
+ echo "| sudo apt -y install libevent-dev libjansson-dev zlib1g-dev cargo
|"
+ echo "|
|"
+ echo "| Rust may be needed too, see https://rustup.rs for the details
|"
+ echo
"+-------------------------------------------------------------------------+"
+ echo
+ echo
+fi
+
+if [ -z ${QUICHE_BSSL_PATH+x} ]; then
+ QUICHE_BSSL_PATH=${TMP_QUICHE_BSSL_PATH:-"${BASE}/boringssl/lib"}
+fi
+
+set -x
+if [ `uname -s` = "Linux" ]
+then
+ num_threads=$(nproc)
+elif [ `uname -s` = "FreeBSD" ]
+then
+ num_threads=$(sysctl -n hw.ncpu)
+else
+ # MacOS.
+ num_threads=$(sysctl -n hw.logicalcpu)
+fi
+
+# boringssl
+echo "Building boringssl..."
+
+# We need this go version.
+mkdir -p ${BASE}/go
+
+if [ `uname -m` = "arm64" -o `uname -m` = "aarch64" ]; then
+ ARCH="arm64"
+else
+ ARCH="amd64"
+fi
+
+if [ `uname -s` = "Darwin" ]; then
+ OS="darwin"
+elif [ `uname -s` = "FreeBSD" ]; then
+ OS="freebsd"
+else
+ OS="linux"
+fi
+
+wget https://go.dev/dl/go1.20.1.${OS}-${ARCH}.tar.gz
+rm -rf ${BASE}/go && tar -C ${BASE} -xf go1.20.1.${OS}-${ARCH}.tar.gz
+rm go1.20.1.${OS}-${ARCH}.tar.gz
+
+GO_BINARY_PATH=${BASE}/go/bin/go
+if [ ! -d boringssl ]; then
+ git clone https://boringssl.googlesource.com/boringssl
+ cd boringssl
+ git checkout 31bad2514d21f6207f3925ba56754611c462a873
+ cd ..
+fi
+cd boringssl
+mkdir -p build
+cd build
+cmake \
+ -DGO_EXECUTABLE=${GO_BINARY_PATH} \
+ -DCMAKE_INSTALL_PREFIX=${BASE}/boringssl \
+ -DCMAKE_BUILD_TYPE=Release \
+ -DBUILD_SHARED_LIBS=1 ../
+
+${MAKE} -j ${num_threads}
+${MAKE} install
+cd ../..
+
+# Build quiche
+# Steps borrowed from:
https://github.com/apache/trafficserver-ci/blob/main/docker/rockylinux8/Dockerfile
+echo "Building quiche"
+QUICHE_BASE="${BASE:-/opt}/quiche"
+[ ! -d quiche ] && git clone --recursive
https://github.com/cloudflare/quiche.git
+cd quiche
+# Latest quiche commits breaks our code so we build from the last commit
+# we know it works, in this case this commit includes the rpath fix commit
+# for quiche. https://github.com/cloudflare/quiche/pull/1508
+# Why does the latest break our code? ->
https://github.com/cloudflare/quiche/pull/1537
+git checkout a1b212761c6cc0b77b9121cdc313e507daf6deb3
+QUICHE_BSSL_PATH=${QUICHE_BSSL_PATH} QUICHE_BSSL_LINK_KIND=dylib cargo build
-j4 --package quiche --release --features ffi,pkg-config-meta,qlog
+mkdir -p ${QUICHE_BASE}/lib/pkgconfig
+mkdir -p ${QUICHE_BASE}/include
+cp target/release/libquiche.a ${QUICHE_BASE}/lib/
+[ -f target/release/libquiche.so ] && cp target/release/libquiche.so
${QUICHE_BASE}/lib/
+cp quiche/include/quiche.h ${QUICHE_BASE}/include/
+cp target/release/quiche.pc ${QUICHE_BASE}/lib/pkgconfig
+cd ..
+
+# OpenSSL needs special hackery ... Only grabbing the branch we need here...
Bryan has shit for network.
+echo "Building OpenSSL with QUIC support"
+[ ! -d openssl-quic ] && git clone -b ${OPENSSL_BRANCH} --depth 1
https://github.com/quictls/openssl.git openssl-quic
+cd openssl-quic
+git checkout 6c41837e9234a8c250f02ae8aa30f44e91342ef6
+./config enable-tls1_3 --prefix=${OPENSSL_PREFIX}
+${MAKE} -j ${num_threads}
+${MAKE} install_sw
+
+# The symlink target provides a more convenient path for the user while also
+# providing, in the symlink source, the precise branch of the OpenSSL build.
+ln -sf ${OPENSSL_PREFIX} ${OPENSSL_BASE}
+cd ..
+
+# OpenSSL will install in /lib or lib64 depending upon the architecture.
+if [ -f "${OPENSSL_PREFIX}/lib/libssl.so" ]; then
+ OPENSSL_LIB="${OPENSSL_PREFIX}/lib"
+elif [ -f "${OPENSSL_PREFIX}/lib64/libssl.so" ]; then
+ OPENSSL_LIB="${OPENSSL_PREFIX}/lib64"
+else
+ echo "Could not find the OpenSSL install library directory."
+ exit 1
+fi
+LDFLAGS=${LDFLAGS:-"-Wl,-rpath,${OPENSSL_LIB}"}
+
+# Then nghttp3
+echo "Building nghttp3..."
+if [ ! -d nghttp3 ]; then
+ git clone --depth 1 -b v0.12.0 https://github.com/ngtcp2/nghttp3.git
+ cd nghttp3
+ cd ..
+fi
+cd nghttp3
+autoreconf -if
+./configure \
+ --prefix=${BASE} \
+ PKG_CONFIG_PATH=${BASE}/lib/pkgconfig:${OPENSSL_LIB}/pkgconfig \
+ CFLAGS="${CFLAGS}" \
+ CXXFLAGS="${CXXFLAGS}" \
+ LDFLAGS="${LDFLAGS}" \
+ --enable-lib-only
+${MAKE} -j ${num_threads}
+${MAKE} install
+cd ..
+
+# Now ngtcp2
+echo "Building ngtcp2..."
+if [ ! -d ngtcp2 ]; then
+ git clone --depth 1 -b v0.16.0 https://github.com/ngtcp2/ngtcp2.git
+ cd ngtcp2
+ cd ..
+fi
+cd ngtcp2
+autoreconf -if
+./configure \
+ --prefix=${BASE} \
+ PKG_CONFIG_PATH=${BASE}/lib/pkgconfig:${OPENSSL_LIB}/pkgconfig \
+ CFLAGS="${CFLAGS}" \
+ CXXFLAGS="${CXXFLAGS}" \
+ LDFLAGS="${LDFLAGS}" \
+ --enable-lib-only
+${MAKE} -j ${num_threads}
+${MAKE} install
+cd ..
+
+# Then nghttp2, with support for H3
+echo "Building nghttp2 ..."
+if [ ! -d nghttp2 ]; then
+ git clone https://github.com/tatsuhiro-t/nghttp2.git
+ cd nghttp2
+ # The following has a fix for builds on systems, like Mac, which do not have
+ # libev. There isn't currently a release with this fix yet.
+ git checkout 2c955ab76b42dfce58e812da6bbe8a526a125fea
+ cd ..
+fi
+cd nghttp2
+autoreconf -if
+if [ `uname -s` = "Darwin" ] || [ `uname -s` = "FreeBSD" ]
+then
+ # --enable-app requires systemd which is not available on Mac/FreeBSD.
+ ENABLE_APP=""
+else
+ ENABLE_APP="--enable-app"
+fi
+
+# Note for FreeBSD: This will not build h2load. h2load can be run on a remote
machine.
+./configure \
+ --prefix=${BASE} \
+ PKG_CONFIG_PATH=${BASE}/lib/pkgconfig:${OPENSSL_LIB}/pkgconfig \
+ CFLAGS="${CFLAGS}" \
+ CXXFLAGS="${CXXFLAGS}" \
+ LDFLAGS="${LDFLAGS}" \
+ --enable-http3 \
+ ${ENABLE_APP}
+${MAKE} -j ${num_threads}
+${MAKE} install
+cd ..
+
+# Then curl
+echo "Building curl ..."
+[ ! -d curl ] && git clone https://github.com/curl/curl.git
+cd curl
+# There isn't currently a released curl yet which has the updates for the above
+# ngtcp2 and nghttp3 library versions.
+git checkout 891e25edb8527bb8de79cdca6d943216c230e905
+# On mac autoreconf fails on the first attempt with an issue finding ltmain.sh.
+# The second runs fine.
+autoreconf -fi || autoreconf -fi
+./configure \
+ --prefix=${BASE} \
+ --with-ssl=${OPENSSL_PREFIX} \
+ --with-nghttp2=${BASE} \
+ --with-nghttp3=${BASE} \
+ --with-ngtcp2=${BASE} \
+ CFLAGS="${CFLAGS}" \
+ CXXFLAGS="${CXXFLAGS}" \
+ LDFLAGS="${LDFLAGS}"
+${MAKE} -j ${num_threads}
+${MAKE} install
+cd ..
