This is an automated email from the ASF dual-hosted git repository.
pdallig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new e70b57c9ba [ZEPPELIN-5819] Configures ivy to download jar libraries
only from remote
e70b57c9ba is described below
commit e70b57c9badfa38263cc85ac62293e2ce0e2d49f
Author: Philipp Dallig <[email protected]>
AuthorDate: Fri Oct 21 15:41:05 2022 +0200
[ZEPPELIN-5819] Configures ivy to download jar libraries only from remote
### What is this PR for?
Configures ivy to download jar libraries only from remote and ignore local
jars
### What type of PR is it?
* Hot Fix
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-5819
### How should this be tested?
* CI
### Questions:
* Does the licenses files need to update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Philipp Dallig <[email protected]>
Closes #4495 from Reamer/ivy_download and squashes the following commits:
b74631ab3 [Philipp Dallig] Configures ivy to download jar libraries only
from remote
---
.../zeppelin/integration/SparkIntegrationTest.java | 19 ++++++++++++++++++-
.../src/test/resources/ivysettings.xml | 6 ++++++
2 files changed, 24 insertions(+), 1 deletion(-)
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java
index cdc040feea..94c9188c8a 100644
---
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java
@@ -17,6 +17,7 @@
package org.apache.zeppelin.integration;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
@@ -46,6 +47,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.EnumSet;
import java.util.List;
import java.util.stream.Collectors;
@@ -101,6 +103,18 @@ public abstract class SparkIntegrationTest {
// sub class can customize spark interpreter setting.
}
+ /**
+ * Configures ivy to download jar libraries only from remote.
+ *
+ * @param interpreterSetting
+ * @throws IOException
+ */
+ private void setupIvySettings(InterpreterSetting interpreterSetting) throws
IOException {
+ File ivysettings = new File(zeppelin.getZeppelinConfDir(),
"ivysettings.xml");
+
FileUtils.copyToFile(SparkIntegrationTest.class.getResourceAsStream("/ivysettings.xml"),
ivysettings);
+ interpreterSetting.setProperty("spark.jars.ivySettings",
ivysettings.getAbsolutePath());
+ }
+
private boolean isHadoopVersionMatch() {
String version = org.apache.hadoop.util.VersionInfo.getVersion();
String majorVersion = version.split("\\.")[0];
@@ -184,6 +198,7 @@ public abstract class SparkIntegrationTest {
try {
setUpSparkInterpreterSetting(sparkInterpreterSetting);
+ setupIvySettings(sparkInterpreterSetting);
testInterpreterBasics();
// no yarn application launched
@@ -215,6 +230,7 @@ public abstract class SparkIntegrationTest {
try {
setUpSparkInterpreterSetting(sparkInterpreterSetting);
+ setupIvySettings(sparkInterpreterSetting);
testInterpreterBasics();
// 1 yarn application launched
@@ -270,6 +286,7 @@ public abstract class SparkIntegrationTest {
String yarnAppId = null;
try {
setUpSparkInterpreterSetting(sparkInterpreterSetting);
+ setupIvySettings(sparkInterpreterSetting);
testInterpreterBasics();
// 1 yarn application launched
@@ -366,6 +383,6 @@ public abstract class SparkIntegrationTest {
if (process.waitFor() != 0) {
throw new RuntimeException("Fail to run command: which python.");
}
- return IOUtils.toString(process.getInputStream()).trim();
+ return IOUtils.toString(process.getInputStream(),
StandardCharsets.UTF_8).trim();
}
}
diff --git
a/zeppelin-interpreter-integration/src/test/resources/ivysettings.xml
b/zeppelin-interpreter-integration/src/test/resources/ivysettings.xml
new file mode 100644
index 0000000000..3b872e1aa1
--- /dev/null
+++ b/zeppelin-interpreter-integration/src/test/resources/ivysettings.xml
@@ -0,0 +1,6 @@
+<ivysettings>
+ <settings defaultResolver="central"/>
+ <resolvers>
+ <ibiblio name="central" m2compatible="true"
root="https://repo1.maven.org/maven2/"/>
+ </resolvers>
+</ivysettings>
