This is an automated email from the ASF dual-hosted git repository.
sebwrede pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/systemds.git
The following commit(s) were added to refs/heads/main by this push:
new 755f879419 [SYSTEMDS-3280] Docker Setup For HE
755f879419 is described below
commit 755f879419eccad29961d80535caab434f592aea
Author: sebwrede <[email protected]>
AuthorDate: Tue Oct 11 08:02:25 2022 +0200
[SYSTEMDS-3280] Docker Setup For HE
This commit changes the dockerfile for tests to include SEAL for running
the homomorphic encryption tests.
The ignore tags are removed from the HE tests and the tests are added to
the GitHub test workflows.
Closes #1721.
---
.github/workflows/functionsTests.yml | 2 +-
docker/entrypoint.sh | 2 ++
docker/testsysds.Dockerfile | 18 +++++++++++++++---
src/main/java/org/apache/sysds/utils/NativeHelper.java | 9 +++++----
.../paramserv/EncryptedFederatedParamservTest.java | 5 +----
.../functions/homomorphicEncryption/InOutTest.java | 10 ++--------
6 files changed, 26 insertions(+), 20 deletions(-)
diff --git a/.github/workflows/functionsTests.yml
b/.github/workflows/functionsTests.yml
index ae1708b6b7..2a078ceb7d 100644
--- a/.github/workflows/functionsTests.yml
+++ b/.github/workflows/functionsTests.yml
@@ -65,7 +65,7 @@ jobs:
"**.functions.dnn.**,**.functions.paramserv.**",
"**.functions.recompile.**,**.functions.misc.**,**.functions.mlcontext.**",
"**.functions.nary.**,**.functions.quaternary.**",
-
"**.functions.parfor.**,**.functions.pipelines.**,**.functions.privacy.**",
+
"**.functions.parfor.**,**.functions.pipelines.**,**.functions.privacy.**",
"**.functions.homomorphicEncryption.**"
"**.functions.unary.scalar.**,**.functions.updateinplace.**,**.functions.vect.**",
"**.functions.reorg.**,**.functions.rewrite.**,**.functions.ternary.**,**.functions.transform.**",
"**.functions.unary.matrix.**,**.functions.linearization.**,**.functions.jmlc.**"
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 706bdbf087..f2bf344765 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -22,6 +22,8 @@
# A script to execute the tests inside the docker container.
+cd /github/workspace/src/main/cpp
+./build.sh
cd /github/workspace
export MAVEN_OPTS="-Xmx512m -XX:MaxPermSize=128m"
diff --git a/docker/testsysds.Dockerfile b/docker/testsysds.Dockerfile
index 373548f35c..68bc845711 100644
--- a/docker/testsysds.Dockerfile
+++ b/docker/testsysds.Dockerfile
@@ -33,6 +33,7 @@ ENV PATH $JAVA_HOME/bin:$MAVEN_HOME/bin:$PATH
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
ENV LANG en_US.UTF-8
+ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib/
COPY ./src/test/scripts/installDependencies.R installDependencies.R
COPY ./docker/entrypoint.sh /entrypoint.sh
@@ -61,8 +62,12 @@ RUN apt-get update -qq \
https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.13%2B8/OpenJDK11U-jdk_x64_linux_11.0.13_8.tar.gz
| tar xzf - \
&& mv openjdk-11.0.13_8 /usr/lib/jvm/java-11-openjdk-amd64 \
&& wget -qO- \
-http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz
| tar xzf - \
- && mv apache-maven-$MAVEN_VERSION /usr/lib/mvn
+
http://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz
| tar xzf - \
+ && mv apache-maven-$MAVEN_VERSION /usr/lib/mvn \
+ && apt-get install -y --no-install-recommends \
+ git \
+ cmake \
+ patchelf
RUN apt-get install -y --no-install-recommends \
libssl-dev \
@@ -74,4 +79,11 @@ RUN apt-get install -y --no-install-recommends \
&& rm -rf installDependencies.R \
&& rm -rf /var/lib/apt/lists/*
-ENTRYPOINT ["/entrypoint.sh"]
\ No newline at end of file
+# SEAL
+RUN wget -qO-
https://github.com/microsoft/SEAL/archive/refs/tags/v3.7.0.tar.gz | tar xzf - \
+ && cd SEAL-3.7.0 \
+ && cmake -S . -B build -DBUILD_SHARED_LIBS=ON \
+ && cmake --build build \
+ && cmake --install build
+
+ENTRYPOINT ["/entrypoint.sh"]
diff --git a/src/main/java/org/apache/sysds/utils/NativeHelper.java
b/src/main/java/org/apache/sysds/utils/NativeHelper.java
index e86bd56b19..f673ba4e4e 100644
--- a/src/main/java/org/apache/sysds/utils/NativeHelper.java
+++ b/src/main/java/org/apache/sysds/utils/NativeHelper.java
@@ -333,6 +333,7 @@ public class NativeHelper {
* @return true if successfully loaded BLAS
*/
public static boolean loadLibraryHelperFromResource(String libFileName)
{
+ LOG.info("Loading JNI shared library: " + libFileName);
try(InputStream in =
NativeHelper.class.getResourceAsStream("/lib/"+ libFileName)) {
// This logic is added because Java does not allow to
load library from a resource file.
if(in != null) {
@@ -346,10 +347,10 @@ public class NativeHelper {
return true;
}
else
- LOG.warn("No lib available in the jar:" +
libFileName);
+ LOG.error("No lib available in the jar:" +
libFileName);
}
- catch(IOException e) {
- LOG.warn("Unable to load library " + libFileName + "
from resource:" + e.getMessage());
+ catch(IOException | UnsatisfiedLinkError e) {
+ LOG.error("Unable to load library " + libFileName + "
from resource:" + e.getMessage());
}
return false;
}
@@ -415,4 +416,4 @@ public class NativeHelper {
// different tradeoffs. In current implementation, we always use
GetPrimitiveArrayCritical as it has proven to be
// fastest. We can revisit this decision later and hence I would not
recommend removing this method.
private static native void setMaxNumThreads(int numThreads);
-}
\ No newline at end of file
+}
diff --git
a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java
b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java
index 2a6bbd950d..437869209c 100644
---
a/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/federated/paramserv/EncryptedFederatedParamservTest.java
@@ -32,7 +32,6 @@ import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
import org.apache.sysds.utils.Statistics;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -73,7 +72,7 @@ public class EncryptedFederatedParamservTest extends
AutomatedTestBase {
//{"TwoNN", 5, 1000, 100, 1, 0.01, "BSP",
"BATCH", "KEEP_DATA_ON_WORKER", "NONE", "true", "BALANCED",
200},
{"TwoNN", 2, 4, 1, 4, 0.01,
"SBP", "BATCH", "KEEP_DATA_ON_WORKER", "BASELINE", "false",
"IMBALANCED", 200},
{"TwoNN", 2, 4, 1, 4, 0.01,
"SBP", "BATCH", "KEEP_DATA_ON_WORKER", "BASELINE", "false",
"BALANCED", 200},
- {"CNN", 2, 4, 1, 4, 0.01,
"SBP", "EPOCH", "SHUFFLE", "BASELINE",
"false", "BALANCED", 200},
+ //{"CNN", 2, 4, 1, 4, 0.01,
"SBP", "EPOCH", "SHUFFLE", "BASELINE",
"false", "BALANCED", 200},
/*
// runtime balancing
@@ -127,13 +126,11 @@ public class EncryptedFederatedParamservTest extends
AutomatedTestBase {
}
@Test
- @Ignore
public void EncryptedfederatedParamservSingleNode() {
EncryptedfederatedParamserv(ExecMode.SINGLE_NODE, true);
}
@Test
- @Ignore
public void EncryptedfederatedParamservHybrid() {
EncryptedfederatedParamserv(ExecMode.HYBRID, true);
}
diff --git
a/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java
b/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java
index 5bb952d9ed..0d05a57ad3 100644
---
a/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java
+++
b/src/test/java/org/apache/sysds/test/functions/homomorphicEncryption/InOutTest.java
@@ -19,6 +19,7 @@
package org.apache.sysds.test.functions.homomorphicEncryption;
+import org.apache.sysds.api.DMLException;
import org.apache.sysds.common.Types;
import org.apache.sysds.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysds.runtime.controlprogram.paramserv.NativeHEHelper;
@@ -33,7 +34,6 @@ import org.apache.sysds.runtime.meta.MetaDataFormat;
import org.apache.sysds.test.AutomatedTestBase;
import org.apache.sysds.test.TestConfiguration;
import org.apache.sysds.test.TestUtils;
-import org.junit.Ignore;
import org.junit.Test;
public class InOutTest extends AutomatedTestBase {
@@ -49,18 +49,12 @@ public class InOutTest extends AutomatedTestBase {
@Override
public void setUp() {
- try {
- NativeHEHelper.initialize();
- } catch (Exception e) {
- throw e;
- }
-
+ NativeHEHelper.initialize();
TestUtils.clearAssertionInformation();
addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR,
TEST_NAME, new String[] { "C" }) );
}
@Test
- @Ignore
public void endToEndTest() {
SEALServer server = new SEALServer();
