This is an automated email from the ASF dual-hosted git repository.
jongyoul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 3de1a49140 [ZEPPELIN-5902] Support Spark 3.4 (#4586)
3de1a49140 is described below
commit 3de1a49140495f637de37e01cc030d2e4498a375
Author: Jeff Zhang <[email protected]>
AuthorDate: Mon Jul 3 20:08:34 2023 +0800
[ZEPPELIN-5902] Support Spark 3.4 (#4586)
* [ZEPPELIN-5902] Support Spark 3.4
* Address comment
---
.github/workflows/core.yml | 4 ++
spark/interpreter/pom.xml | 15 +++++-
.../zeppelin/spark/SparkSqlInterpreterTest.java | 6 ---
.../org/apache/zeppelin/spark/SparkVersion.java | 4 +-
.../integration/SparkIntegrationTest34.java | 56 ++++++++++++++++++++++
.../integration/ZeppelinSparkClusterTest34.java | 39 +++++++++++++++
6 files changed, 114 insertions(+), 10 deletions(-)
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index d4cb767c90..ba5feee7a7 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -413,6 +413,10 @@ jobs:
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Phadoop3
-Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ - name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python
}}
+ run: |
+ rm -rf spark/interpreter/metastore_db
+ ./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Phadoop3
-Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
livy-0-7-with-spark-2-2-0-under-python3:
runs-on: ubuntu-20.04
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index 351172343d..a7284bf479 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -518,12 +518,23 @@
</profile>
<!-- profile spark-x only affect spark version used in test -->
-
<profile>
- <id>spark-3.3</id>
+ <id>spark-3.4</id>
<activation>
<activeByDefault>true</activeByDefault>
</activation>
+ <properties>
+ <datanucleus.core.version>4.1.17</datanucleus.core.version>
+ <datanucleus.rdbms.version>4.1.19</datanucleus.rdbms.version>
+ <datanucleus.apijdo.version>4.2.4</datanucleus.apijdo.version>
+ <spark.version>3.4.0</spark.version>
+ <protobuf.version>3.21.12</protobuf.version>
+ <py4j.version>0.10.9.7</py4j.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>spark-3.3</id>
<properties>
<datanucleus.core.version>4.1.17</datanucleus.core.version>
<datanucleus.rdbms.version>4.1.19</datanucleus.rdbms.version>
diff --git
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
index c4a1d8e7d0..433663b7b3 100644
---
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
+++
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkSqlInterpreterTest.java
@@ -309,11 +309,5 @@ public class SparkSqlInterpreterTest {
assertEquals(InterpreterResult.Code.ERROR, ret.code());
assertEquals(1, context.out.toInterpreterResultMessage().size());
assertEquals(Type.TEXT,
context.out.toInterpreterResultMessage().get(0).getType());
-
- // spark 1.x could not detect the root cause correctly
- if (!sparkInterpreter.getSparkContext().version().startsWith("1.")) {
-
assertTrue(context.out.toInterpreterResultMessage().get(0).getData().contains("ClassNotFoundException")
||
-
context.out.toInterpreterResultMessage().get(0).getData().contains("Can not
load class"));
- }
}
}
diff --git
a/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
b/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
index 8f88c1b6e1..7d76f40b74 100644
---
a/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
+++
b/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
@@ -35,10 +35,10 @@ public class SparkVersion {
public static final SparkVersion SPARK_3_3_0 =
SparkVersion.fromVersionString("3.3.0");
- public static final SparkVersion SPARK_3_4_0 =
SparkVersion.fromVersionString("3.4.0");
+ public static final SparkVersion SPARK_3_5_0 =
SparkVersion.fromVersionString("3.5.0");
public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_2_0_0;
- public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_3_4_0;
+ public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_3_5_0;
private int version;
private int majorVersion;
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java
new file mode 100644
index 0000000000..89cdce77c5
--- /dev/null
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest34.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.integration;
+
+import org.apache.zeppelin.interpreter.InterpreterSetting;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.List;
+
+@RunWith(value = Parameterized.class)
+public class SparkIntegrationTest34 extends SparkIntegrationTest {
+
+ public SparkIntegrationTest34(String sparkVersion, String hadoopVersion) {
+ super(sparkVersion, hadoopVersion);
+ }
+
+ @Parameterized.Parameters
+ public static List<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {"3.4.0", "3"},
+ });
+ }
+
+ @Override
+ protected void setUpSparkInterpreterSetting(InterpreterSetting
interpreterSetting) {
+ // spark3 doesn't support yarn-client and yarn-cluster anymore, use
+ // spark.master and spark.submit.deployMode instead
+ String sparkMaster =
interpreterSetting.getJavaProperties().getProperty("spark.master");
+ if (sparkMaster.equals("yarn-client")) {
+ interpreterSetting.setProperty("spark.master", "yarn");
+ interpreterSetting.setProperty("spark.submit.deployMode", "client");
+ } else if (sparkMaster.equals("yarn-cluster")){
+ interpreterSetting.setProperty("spark.master", "yarn");
+ interpreterSetting.setProperty("spark.submit.deployMode", "cluster");
+ } else if (sparkMaster.startsWith("local")) {
+ interpreterSetting.setProperty("spark.submit.deployMode", "client");
+ }
+ }
+}
diff --git
a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest34.java
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest34.java
new file mode 100644
index 0000000000..1581dca482
--- /dev/null
+++
b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/ZeppelinSparkClusterTest34.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.integration;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+import java.util.List;
+
+@RunWith(value = Parameterized.class)
+public class ZeppelinSparkClusterTest34 extends ZeppelinSparkClusterTest {
+
+ public ZeppelinSparkClusterTest34(String sparkVersion, String hadoopVersion)
throws Exception {
+ super(sparkVersion, hadoopVersion);
+ }
+
+ @Parameterized.Parameters
+ public static List<Object[]> data() {
+ return Arrays.asList(new Object[][]{
+ {"3.4.0", "3"}
+ });
+ }
+}
