This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 9c96e1c7e4 [SYSTEMDS-3877] Docker image size reduction and safety
improvement
9c96e1c7e4 is described below
commit 9c96e1c7e448ca716b0a3005bbc30906070ceae4
Author: Alexander Schmidt <[email protected]>
AuthorDate: Tue Oct 7 16:48:27 2025 +0200
[SYSTEMDS-3877] Docker image size reduction and safety improvement
This commit modifies our docker images to follow
current standard procedures to make safe and small
docker images.
In specific the size of our release image is reduced
from 1.5 GB to 450MB, and similarly our testing image
is reduced.
Closes #2274
---
.gitignore | 4 ++
docker/build.sh | 4 +-
docker/sysds.Dockerfile | 71 ++++++++++++++++++++++++++++++------
docker/testsysds.Dockerfile | 89 ++++++++++++++++++++++-----------------------
4 files changed, 108 insertions(+), 60 deletions(-)
diff --git a/.gitignore b/.gitignore
index f3c28571bd..d4b817a1ae 100644
--- a/.gitignore
+++ b/.gitignore
@@ -150,3 +150,7 @@ venv/*
# resource optimization
scripts/resource/output
*.pem
+
+# docker tests
+docker/mountFolder/*.bin
+docker/mountFolder/*.bin.mtd
diff --git a/docker/build.sh b/docker/build.sh
index 2898effdc2..1577051d15 100755
--- a/docker/build.sh
+++ b/docker/build.sh
@@ -23,10 +23,10 @@
# Build the docker containers
# The first build is for running systemds through docker.
-# docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest .
+docker image build -f docker/sysds.Dockerfile -t apache/systemds:latest .
# The second build is for testing systemds. This image installs the R
dependencies needed to run the tests.
-docker image build -f docker/testsysds.Dockerfile -t
apache/systemds:testing-latest .
+# docker image build -f docker/testsysds.Dockerfile -t
apache/systemds:testing-latest .
# The third build is python docker for systemds.
# docker image build -f docker/pythonsysds.Dockerfile -t
apache/systemds:python-nightly .
diff --git a/docker/sysds.Dockerfile b/docker/sysds.Dockerfile
index cc6ef605d5..2238aa878a 100644
--- a/docker/sysds.Dockerfile
+++ b/docker/sysds.Dockerfile
@@ -19,18 +19,16 @@
#
#-------------------------------------------------------------
-FROM
ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233
+FROM
alpine:3.20@sha256:de4fe7064d8f98419ea6b49190df1abbf43450c1702eeb864fe9ced453c1cc5f
AS compile-image
WORKDIR /usr/src/
# Do basic updates on the image
-RUN apt-get update -qq \
- && apt-get upgrade -y \
- && apt-get install -y --no-install-recommends \
+RUN apk add --no-cache \
wget \
git \
ca-certificates \
- && apt-get clean
+ bash
# Set environment variables
# Maven
@@ -43,11 +41,11 @@ ENV SYSTEMDS_ROOT=/usr/src/systemds
ENV PATH=$SYSTEMDS_ROOT/bin:$PATH
ENV SYSDS_QUIET=1
-# Download Java and Mvn
+# Download Mvn and JDK
RUN mkdir -p /usr/lib/jvm \
&& wget -qO- \
-
https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_linux_hotspot_17.0.15_6.tar.gz
| tar xzf - \
- && mv jdk-17.0.15+6 /usr/lib/jvm/jdk-17.0.15+6 \
+
https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_alpine-linux_hotspot_17.0.15_6.tar.gz
| tar xzf - \
+ && mv jdk-17.0.15+6 $JAVA_HOME \
&& wget -qO- \
http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz
| tar xzf - \
&& mv apache-maven-$MAVEN_VERSION /usr/lib/mvn
@@ -57,8 +55,11 @@ RUN git clone --depth 1
https://github.com/apache/systemds.git systemds && \
cd /usr/src/systemds/ && \
mvn --no-transfer-progress clean package -P distribution
+COPY docker/mountFolder/main.dml /input/main.dml
+
# Remove all unnecessary files from the Image
-RUN rm -rf .git && \
+RUN cd /usr/src/systemds/ && \
+ rm -rf .git && \
rm -rf .github && \
rm -rf target/javadoc** && \
rm -rf target/apidocs** && \
@@ -71,9 +72,55 @@ RUN rm -rf .git && \
rm -rf /usr/lib/mvn && \
rm -rf CONTRIBUTING.md && \
rm -rf pom.xml && \
- rm -rf ~/.m2
+ rm -rf ~/.m2 && \
+ rm -rf docker && \
+ rm -rf .mvn
+FROM
alpine:3.20@sha256:de4fe7064d8f98419ea6b49190df1abbf43450c1702eeb864fe9ced453c1cc5f
-COPY docker/mountFolder/main.dml /input/main.dml
+RUN apk add --no-cache bash \
+ snappy \
+ lz4 \
+ zlib
+
+ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6
+ENV PATH=$JAVA_HOME/bin:$PATH
+ENV SYSTEMDS_ROOT=/systemds
+ENV PATH=$SYSTEMDS_ROOT/bin:$PATH
+ENV SYSDS_QUIET=1
+
+ENV HADOOP_VERSION=3.3.6
+ENV HADOOP_HOME=/opt/hadoop
+ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native
+ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
+ENV GLIBC_VERSION=2.35-r1
+
+RUN wget -q -O /etc/apk/keys/sgerrand.rsa.pub
https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub \
+ && wget
https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk
\
+ && apk add glibc-${GLIBC_VERSION}.apk \
+ && rm glibc-${GLIBC_VERSION}.apk
+
+RUN mkdir -p /usr/lib/jvm \
+ && wget -qO- \
+
https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jre_x64_alpine-linux_hotspot_17.0.15_6.tar.gz
| tar xzf - \
+ && mv jdk-17.0.15+6-jre $JAVA_HOME
+
+RUN mkdir -p $HADOOP_HOME/lib/native \
+ && wget -q
https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
&& \
+ tar --strip-components=2 -xzf hadoop-${HADOOP_VERSION}.tar.gz \
+ hadoop-${HADOOP_VERSION}/lib/native && \
+ mv native/libhadoop.so.1.0.0 /opt/hadoop/lib/native && \
+ mv native/libhadoop.so /opt/hadoop/lib/native && \
+ rm hadoop-${HADOOP_VERSION}.tar.gz && \
+ rm -rf native
+
+COPY --from=compile-image /usr/src/systemds /systemds
+COPY --from=compile-image /input/main.dml /input/main.dml
+
+WORKDIR /input
+
+RUN addgroup -S default && adduser -S systemds -G default
+USER systemds
-CMD ["systemds", "/input/main.dml"]
+ENTRYPOINT ["systemds"]
+CMD ["main.dml"]
diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile
index 2f63dace7f..fc6eb1491e 100644
--- a/docker/testsysds.Dockerfile
+++ b/docker/testsysds.Dockerfile
@@ -18,8 +18,29 @@
# under the License.
#
#-------------------------------------------------------------
+# Stage 1: Build SEAL
+FROM
debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973
AS seal-build
-FROM
ubuntu:24.04@sha256:6015f66923d7afbc53558d7ccffd325d43b4e249f41a6e93eef074c9505d2233
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ build-essential \
+ cmake \
+ wget \
+ tar \
+ git \
+ ca-certificates \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /seal
+
+# Install SEAL
+RUN wget -qO-
https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \
+ && cd SEAL-3.7.0 \
+ && cmake -S . -B build -DBUILD_SHARED_LIBS=ON \
+ && cmake --build build \
+ && cmake --install build --prefix /seal-install
+
+# Stage 2: Final image with R, JDK, Maven, SEAL
+FROM
debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973
WORKDIR /usr/src/
ENV MAVEN_VERSION=3.9.9
@@ -28,61 +49,37 @@ ENV MAVEN_HOME=/usr/lib/mvn
ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6
ENV PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-ENV LANG=en_US.UTF-8
-ENV LD_LIBRARY_PATH=/usr/local/lib/
-
-RUN apt-get update -qq \
- && apt-get upgrade -y \
- && apt-get install -y --no-install-recommends \
- libcurl4-openssl-dev \
- libxml2-dev \
- locales \
- software-properties-common \
- dirmngr \
- gnupg \
- apt-transport-https \
- wget \
- ca-certificates \
- git \
- cmake \
- patchelf \
- && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys
E298A3A825C0D65DFD57CBB651716619E084DAB9 \
- && add-apt-repository "deb https://cloud.r-project.org/bin/linux/ubuntu
$(lsb_release -cs)-cran40/" \
- && apt-get update -qq \
- && apt-get upgrade -y \
- && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen \
- && locale-gen en_US.utf8 \
- && /usr/sbin/update-locale LANG=en_US.UTF-8 \
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ r-base \
+ wget \
+ cmake \
+ r-base-dev \
+ libcurl4-openssl-dev \
+ libssl-dev \
+ libxml2-dev \
+ ca-certificates \
+ patchelf \
+ git \
+ libssl-dev \
+ r-base-dev \
+ r-base-core \
+ && apt-get clean && rm -rf /var/lib/apt/lists/* \
&& mkdir -p /usr/lib/jvm \
&& wget -qO- \
https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.15%2B6/OpenJDK17U-jdk_x64_linux_hotspot_17.0.15_6.tar.gz
| tar xzf - \
- && mv jdk-17.0.15+6 /usr/lib/jvm/jdk-17.0.15+6 \
+ && mv jdk-17.0.15+6 $JAVA_HOME \
&& wget -qO- \
-
http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz
| tar xzf - \
+
http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz
| tar xzf - \
&& mv apache-maven-$MAVEN_VERSION /usr/lib/mvn
-# Install R Base
-RUN apt-get install -y --no-install-recommends \
- libssl-dev \
- r-base \
- r-base-dev \
- r-base-core
-
# Install R packages
-COPY ./src/test/scripts/installDependencies.R installDependencies.R
+COPY ./src/test/scripts/installDependencies.R installDependencies.R
RUN Rscript installDependencies.R \
- && rm -rf installDependencies.R \
- && rm -rf /var/lib/apt/lists/*
+ && rm -f installDependencies.R
-# Install SEAL
-RUN wget -qO-
https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \
- && cd SEAL-3.7.0 \
- && cmake -S . -B build -DBUILD_SHARED_LIBS=ON \
- && cmake --build build \
- && cmake --install build
+# Copy SEAL
+COPY --from=seal-build /seal-install /usr/local
# Finally copy the entrypoint script
# This is last to enable quick updates to the script after initial local build.
