This is an automated email from the ASF dual-hosted git repository.
baunsgaard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 0736c68b1c [SYSTEMDS-3877] Docker cleanup
0736c68b1c is described below
commit 0736c68b1cca5fd35639171ef75a37fda819aaea
Author: Sebastian Baunsgaard <[email protected]>
AuthorDate: Wed Oct 8 22:25:30 2025 +0200
[SYSTEMDS-3877] Docker cleanup
This commit fixes three bugs in the new testing image.
1. The seal library install path, to be included in the library path
2. The locale specification to US English UTF-8, to allow string
comparison of Hadoop read files, without casting from 16 to 8.
Furthermore, the commit does a few maintainable cleanups in the
testing to reduce verbose logging of our test cases, and allow
non-buffered writing of cloud running tests, to enable easy live
tracking of GitHub action tests.
Closes #2335
---
.gitignore | 3 ++
docker/README.md | 1 +
docker/entrypoint.sh | 19 ++++++------
docker/testsysds.Dockerfile | 34 +++++++++++++++++++---
src/main/cpp/build_HE.sh | 4 +--
src/test/java/org/apache/sysds/test/TestUtils.java | 10 +++++++
.../transform/TransformCSVFrameEncodeReadTest.java | 8 ++---
7 files changed, 59 insertions(+), 20 deletions(-)
diff --git a/.gitignore b/.gitignore
index d4b817a1ae..d2fcdb9a4d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -154,3 +154,6 @@ scripts/resource/output
# docker tests
docker/mountFolder/*.bin
docker/mountFolder/*.bin.mtd
+
+SEAL-*/
+
diff --git a/docker/README.md b/docker/README.md
index d530061019..42f751618b 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -138,6 +138,7 @@ Test your testing image locally by running the following
command:
```bash
docker run \
-v $(pwd):/github/workspace \
+ -v $HOME/.m2/repository:/root/.m2/repository \
apache/systemds:testing-latest \
org.apache.sysds.test.component.**
```
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index c276a707c3..9a77766b3a 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -29,23 +29,22 @@ cd /github/workspace
export MAVEN_OPTS="-Xmx512m"
log="/tmp/sysdstest.log"
-mvn -ntp -B test-compile 2>&1 | grep -E "BUILD|Total time:|---|Building
SystemDS"
+mvn -ntp -B test-compile 2>&1 | stdbuf -oL grep -E "BUILD|Total
time:|---|Building SystemDS"
mvn -ntp -B test -D maven.test.skip=false -D
automatedtestbase.outputbuffering=true -D test=$1 2>&1 \
- | grep -v "already exists in destination." \
- | grep -v 'WARNING: Using incubator modules' | tee $log
+ | stdbuf -oL grep -Ev "already exists in destination.|Using incubator" \
+ | tee $log
-# Merge Federated test runs.
-[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec
-mvn -ntp -B jacoco:merge
-
-# Merge Federated test runs.
-[ -f target/jacoco.exec ] && mv target/jacoco.exec target/jacoco_main.exec
-mvn -ntp -B jacoco:merge
grep_args="SUCCESS"
grepvals="$( tail -n 100 $log | grep $grep_args)"
if [[ $grepvals == *"SUCCESS"* ]]; then
+ # Merge Federated test runs.
+ # if merged jacoco exist temporarily rename to not overwrite.
+ [ -f target/jacoco.exec ] && mv target/jacoco.exec
target/jacoco_main.exec
+ # merge jacoco files.
+ mvn -ntp -B jacoco:merge 2>&1 | stdbuf -oL grep -E "jacoco"
+
exit 0
else
exit 1
diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile
index fc6eb1491e..a4bfc14416 100644
--- a/docker/testsysds.Dockerfile
+++ b/docker/testsysds.Dockerfile
@@ -19,7 +19,7 @@
#
#-------------------------------------------------------------
# Stage 1: Build SEAL
-FROM
debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973
AS seal-build
+FROM
ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123
AS seal-build
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential \
@@ -40,16 +40,24 @@ RUN wget -qO-
https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz
&& cmake --install build --prefix /seal-install
# Stage 2: Final image with R, JDK, Maven, SEAL
-FROM
debian:bullseye-slim@sha256:b5f9bc44bdfbd9d551dfdd432607cbc6bb5d9d6dea726a1191797d7749166973
+FROM
ubuntu:noble@sha256:728785b59223d755e3e5c5af178fab1be7031f3522c5ccd7a0b32b80d8248123
WORKDIR /usr/src/
ENV MAVEN_VERSION=3.9.9
ENV MAVEN_HOME=/usr/lib/mvn
+ENV LANGUAGE=en_US:en
+ENV LC_ALL=en_US.UTF-8
+ENV LANG=en_US.UTF-8
ENV JAVA_HOME=/usr/lib/jvm/jdk-17.0.15+6
ENV PATH=$JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN apt-get update && apt-get install -y locales \
+ && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \
+ && locale-gen \
+ && update-locale LANG=en_US.UTF-8
+
+RUN apt-get install -y --no-install-recommends \
r-base \
wget \
cmake \
@@ -78,8 +86,26 @@ COPY ./src/test/scripts/installDependencies.R
installDependencies.R
RUN Rscript installDependencies.R \
&& rm -f installDependencies.R
+ENV HADOOP_VERSION=3.3.6
+ENV HADOOP_HOME=/opt/hadoop
+ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native
+ENV HADOOP_OPTS="-Djava.library.path=$HADOOP_HOME/lib/native"
+
+RUN mkdir -p $HADOOP_HOME/lib/native \
+ && wget -q
https://downloads.apache.org/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz
&& \
+ tar --strip-components=2 -xzf hadoop-${HADOOP_VERSION}.tar.gz \
+ hadoop-${HADOOP_VERSION}/lib/native && \
+ mv native/libhadoop.so.1.0.0 /opt/hadoop/lib/native && \
+ mv native/libhadoop.so /opt/hadoop/lib/native && \
+ rm hadoop-${HADOOP_VERSION}.tar.gz && \
+ rm -rf native
+
# Copy SEAL
-COPY --from=seal-build /seal-install /usr/local
+COPY --from=seal-build /seal-install/lib/ /usr/local/lib/
+COPY --from=seal-build /seal-install/include/ /usr/local/include/
+
+ENV LD_LIBRARY_PATH=/opt/hadoop/lib/native;/usr/local/lib/
+
# Finally copy the entrypoint script
# This is last to enable quick updates to the script after initial local build.
diff --git a/src/main/cpp/build_HE.sh b/src/main/cpp/build_HE.sh
index dec3622a42..24be7f7bb7 100755
--- a/src/main/cpp/build_HE.sh
+++ b/src/main/cpp/build_HE.sh
@@ -20,7 +20,7 @@
#
#-------------------------------------------------------------
-echo "Build Homomorphic Encryption Liberary: "
+echo "Build Homomorphic Encryption Library: "
# compile HE
cmake he/ -B HE -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++
@@ -29,4 +29,4 @@ rm -R HE
echo ""
-echo "Sucessfull install of Homomorphic Encryption Liberary SEAL"
\ No newline at end of file
+echo "Successful install of Homomorphic Encryption Library SEAL"
\ No newline at end of file
diff --git a/src/test/java/org/apache/sysds/test/TestUtils.java
b/src/test/java/org/apache/sysds/test/TestUtils.java
index 107d33b1a1..e470dd8253 100644
--- a/src/test/java/org/apache/sysds/test/TestUtils.java
+++ b/src/test/java/org/apache/sysds/test/TestUtils.java
@@ -1760,15 +1760,25 @@ public class TestUtils {
}
public static double[][] convertHashMapToDoubleArray(HashMap
<CellIndex, Double> matrix) {
+ if(matrix.isEmpty()){
+ LOG.error("converting empty hashMap");
+ return new double[0][0];
+ }
int max_rows = -1, max_cols= -1;
+
for(CellIndex ix : matrix.keySet()) {
max_rows = Math.max(max_rows, ix.row);
max_cols = Math.max(max_cols, ix.column);
}
+
return convertHashMapToDoubleArray(matrix, max_rows, max_cols);
}
public static double[][] convertHashMapToDoubleArray(HashMap<CellIndex,
Double> matrix, int rows, int cols) {
+ if(rows <= -1 || cols <= -1){
+ LOG.error("converting negative size hashmap rows: " +
rows + " cols: " + cols);
+ return new double[0][0];
+ }
double [][] ret_arr = new double[rows][cols];
for(Entry<CellIndex, Double> e : matrix.entrySet()) {
int i = e.getKey().row-1;
diff --git
a/src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java
b/src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java
index 41872c3ba5..ce2e7ba062 100644
---
a/src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/transform/TransformCSVFrameEncodeReadTest.java
@@ -148,16 +148,16 @@ public class TransformCSVFrameEncodeReadTest extends
AutomatedTestBase {
String[] fromDisk =
DataConverter.toString(fb2).split("\n");
String[] printed = stdOut.split("\n");
boolean equal = true;
- String err = "";
+ StringBuilder err = new StringBuilder();
for(int i = 0; i < fromDisk.length; i++){
- if(! fromDisk[i].equals(printed[i])){
- err += "\n not equal: \n"+ (fromDisk[i]
+ "\n" + printed[i]);
+ if(!
fromDisk[i].strip().equals(printed[i].strip())){
+ err.append("\n not equal: \n'"+
(fromDisk[i] + "'\n'" + printed[i] + "'"));
equal = false;
}
}
if(!equal)
- fail(err);
+ fail(err.toString());
}
catch(Exception ex) {
