(zeppelin) branch master updated: [ZEPPELIN-6027] Enhanced Integration with Apache Kyuubi (#4770)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 19808172ed [ZEPPELIN-6027] Enhanced Integration with Apache Kyuubi (#4770) 19808172ed is described below commit 19808172ed0dc589ef35e847cb4e9a276769fd77 Author: Cheng Pan AuthorDate: Wed Jul 3 22:11:18 2024 +0800 [ZEPPELIN-6027] Enhanced Integration with Apache Kyuubi (#4770) * [ZEPPELIN-6027] Enhanced Integration with Apache Kyuubi * fix style --- jdbc/pom.xml | 8 ++ .../org/apache/zeppelin/jdbc/JDBCInterpreter.java | 13 ++- .../jdbc/kyuubi/BeelineInPlaceUpdateStream.java| 118 + .../apache/zeppelin/jdbc/kyuubi/KyuubiUtils.java | 118 + .../apache/zeppelin/jdbc/kyuubi/ProgressBar.java | 53 + jdbc/src/main/resources/interpreter-setting.json | 21 6 files changed, 328 insertions(+), 3 deletions(-) diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 35be95c71d..f60b66aae6 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -39,6 +39,7 @@ 2.2.220 2.0.1 3.1.3 +1.9.1 1.0.8 @@ -135,6 +136,13 @@ + + org.apache.kyuubi + kyuubi-hive-jdbc-shaded + ${kyuubi.version} + provided + + net.jodah concurrentunit diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index e31c6e4a5d..b4cfba25b4 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -35,6 +35,7 @@ import org.apache.zeppelin.interpreter.SingleRowInterpreterResult; import org.apache.zeppelin.interpreter.ZeppelinContext; import org.apache.zeppelin.interpreter.util.SqlSplitter; import org.apache.zeppelin.jdbc.hive.HiveUtils; +import org.apache.zeppelin.jdbc.kyuubi.KyuubiUtils; import org.apache.zeppelin.tabledata.TableDataUtils; import org.apache.zeppelin.util.PropertiesUtil; import org.slf4j.Logger; @@ -825,10 +826,16 @@ public class JDBCInterpreter extends KerberosInterpreter { String jdbcURL = getJDBCConfiguration(user).getProperty().getProperty(URL_KEY); String driver = getJDBCConfiguration(user).getProperty().getProperty(DRIVER_KEY); - if (jdbcURL != null && jdbcURL.startsWith("jdbc:hive2://") - && driver != null && driver.equals("org.apache.hive.jdbc.HiveDriver")) { -HiveUtils.startHiveMonitorThread(statement, context, + if (jdbcURL != null && driver != null) { +if (driver.equals("org.apache.hive.jdbc.HiveDriver") && +jdbcURL.startsWith("jdbc:hive2://")) { + HiveUtils.startHiveMonitorThread(statement, context, Boolean.parseBoolean(getProperty("hive.log.display", "true")), this); +} else if (driver.equals("org.apache.kyuubi.jdbc.KyuubiHiveDriver") && +(jdbcURL.startsWith("jdbc:kyuubi://") || jdbcURL.startsWith("jdbc:hive2://"))) { + KyuubiUtils.startMonitorThread(connection, statement, context, + Boolean.parseBoolean(getProperty("kyuubi.log.display", "true")), this); +} } boolean isResultSetAvailable = statement.execute(sqlToExecute); getJDBCConfiguration(user).setConnectionInDBDriverPoolSuccessful(); diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/kyuubi/BeelineInPlaceUpdateStream.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/kyuubi/BeelineInPlaceUpdateStream.java new file mode 100644 index 00..4218461373 --- /dev/null +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/kyuubi/BeelineInPlaceUpdateStream.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License
[zeppelin] branch master updated: [MINOR] Use flink-113 as default profile (#4566)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 67fb2add81 [MINOR] Use flink-113 as default profile (#4566) 67fb2add81 is described below commit 67fb2add819eed6a13bcc1608e69ae59f8f8c323 Author: Jeff Zhang AuthorDate: Fri Feb 17 22:06:27 2023 +0800 [MINOR] Use flink-113 as default profile (#4566) --- flink/pom.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flink/pom.xml b/flink/pom.xml index c0e17389c3..1939c1ccb4 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -79,6 +79,9 @@ flink-113 + +true + flink-scala-2.11 flink-scala-2.12
[zeppelin] branch master updated: [HOTFIX] Fix CI failure due to sqlalchemy api change (#4561)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new c48e35f27d [HOTFIX] Fix CI failure due to sqlalchemy api change (#4561) c48e35f27d is described below commit c48e35f27d417e9d7b6f748e52a2b40304971501 Author: Jeff Zhang AuthorDate: Sat Feb 4 18:53:31 2023 +0800 [HOTFIX] Fix CI failure due to sqlalchemy api change (#4561) --- testing/env_python_3.8_with_R.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/testing/env_python_3.8_with_R.yml b/testing/env_python_3.8_with_R.yml index 34e228e9d0..fd566bbe4d 100644 --- a/testing/env_python_3.8_with_R.yml +++ b/testing/env_python_3.8_with_R.yml @@ -9,6 +9,7 @@ dependencies: - grpcio - protobuf - pandasql + - sqlalchemy=1.4.46 - ipython - ipython_genutils - ipykernel
[zeppelin] branch master updated: [ZEPPELIN-5844] Support flink 1.16 (#4506)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 521acca985 [ZEPPELIN-5844] Support flink 1.16 (#4506) 521acca985 is described below commit 521acca98570b1e64b4c3ae569aad5da8423b0f7 Author: Jeff Zhang AuthorDate: Tue Jan 31 20:17:30 2023 +0800 [ZEPPELIN-5844] Support flink 1.16 (#4506) * [ZEPPELIN-5844] Support flink 1.16 * Update github action * Fix CI * save * Fix flink test * Remove -B in core.yaml --- .github/workflows/core.yml | 14 +- flink/flink-scala-parent/pom.xml | 73 ++- .../org/apache/zeppelin/flink/TableEnvFactory.java | 186 +-- .../internal/ScalaShellStreamEnvironment.java | 10 + .../zeppelin/flink/FlinkScalaInterpreter.scala | 27 +- .../zeppelin/flink/internal/FlinkILoop.scala | 43 +- .../flink/FlinkStreamSqlInterpreterTest.java | 10 +- .../{init_stream.scala => init_stream.scala2} | 0 .../java/org/apache/zeppelin/flink/FlinkShims.java | 31 +- .../org/apache/zeppelin/flink/Flink112Shims.java | 59 ++- .../org/apache/zeppelin/flink/Flink113Shims.java | 59 ++- .../org/apache/zeppelin/flink/Flink114Shims.java | 59 ++- .../org/apache/zeppelin/flink/Flink115Shims.java | 60 ++- flink/flink1.16-shims/pom.xml | 207 .../org/apache/zeppelin/flink/Flink116Shims.java} | 105 +++- .../zeppelin/flink/Flink116SqlInterpreter.java | 590 + .../java/org/apache/zeppelin/flink/PrintUtils.java | 318 +++ .../zeppelin/flink/TimestampStringUtils.java | 143 + .../flink/shims116/CollectStreamTableSink.java | 97 flink/pom.xml | 10 + testing/env_python_3_with_flink_116.yml| 29 + 21 files changed, 1886 insertions(+), 244 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 4d806acf87..76380103da 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -233,7 +233,7 @@ jobs: strategy: fail-fast: false matrix: -flink: [112, 113, 114, 115] +flink: [112, 113, 114, 115, 116] steps: - name: Checkout uses: actions/checkout@v3 @@ -256,12 +256,12 @@ jobs: restore-keys: | ${{ runner.os }}-zeppelin- - name: install environment for flink before 1.15 (exclusive) -if: matrix.flink != '115' +if: matrix.flink < '115' run: | ./mvnw install -DskipTests -am -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration ${MAVEN_ARGS} ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} - name: install environment for flink after 1.15 (inclusive) -if: matrix.flink == '115' +if: matrix.flink >= '115' run: | ./mvnw install -DskipTests -am -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration ${MAVEN_ARGS} ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests ${MAVEN_ARGS} @@ -277,10 +277,10 @@ jobs: auto-activate-base: false use-mamba: true - name: run tests for flink before 1.15 (exclusive) -if: matrix.flink != '115' -run: ./mvnw verify -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -am -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} - - name: run tests for flink before 1.15 (inclusive) -if: matrix.flink == '115' +if: matrix.flink < '115' +run: ./mvnw verify -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} + - name: run tests for flink after 1.15 (inclusive) +if: matrix.flink >= '115' run: ./mvnw verify -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -am -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=org.apache.zeppelin.flink.*Test,FlinkIntegrationTest${{ matrix.flink }} ${MAVEN_ARGS} - name: Print zeppelin logs if: always() diff --git a/flink/flink-scala-parent/pom.xml b/flink/flink-scala-parent/pom.xml index e9f364a162..8bbeebd26f 100644 --- a/flink/flink-scala-parent/pom.xml +++ b/flink/flink-scala-parent/pom.xml @@ -79,6 +79,12 @@ ${
[zeppelin] branch master updated: [ZEPPELIN-5763] NoClassDefFoundError in spark interpreter: org/apache/commons/lang/StringUtils (#4501)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new ac5c9b4c26 [ZEPPELIN-5763] NoClassDefFoundError in spark interpreter: org/apache/commons/lang/StringUtils (#4501) ac5c9b4c26 is described below commit ac5c9b4c26a879e68b3c984f9ee424fbadb655ea Author: Jeff Zhang AuthorDate: Tue Nov 8 11:22:10 2022 +0800 [ZEPPELIN-5763] NoClassDefFoundError in spark interpreter: org/apache/commons/lang/StringUtils (#4501) * [ZEPPELIN-5763] java.lang.NoClassDefFoundError: org/apache/commons/lang/StringUtils * Remove commons-lang --- spark/interpreter/pom.xml | 4 .../java/org/apache/zeppelin/spark/AbstractSparkScalaInterpreter.java | 2 +- .../src/main/java/org/apache/zeppelin/spark/Spark2Shims.java | 2 +- .../src/main/java/org/apache/zeppelin/spark/Spark3Shims.java | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index 87dfe26cbc..eb4ea1f690 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -217,6 +217,10 @@ com.google.protobuf protobuf-java + + commons-lang + commons-lang + diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkScalaInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkScalaInterpreter.java index bc58ea7461..4e83c74ceb 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkScalaInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/AbstractSparkScalaInterpreter.java @@ -18,7 +18,7 @@ package org.apache.zeppelin.spark; import com.google.common.collect.Lists; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; diff --git a/spark/spark2-shims/src/main/java/org/apache/zeppelin/spark/Spark2Shims.java b/spark/spark2-shims/src/main/java/org/apache/zeppelin/spark/Spark2Shims.java index 21fb149ff3..1b5cffb7ad 100644 --- a/spark/spark2-shims/src/main/java/org/apache/zeppelin/spark/Spark2Shims.java +++ b/spark/spark2-shims/src/main/java/org/apache/zeppelin/spark/Spark2Shims.java @@ -18,7 +18,7 @@ package org.apache.zeppelin.spark; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkContext; import org.apache.spark.scheduler.SparkListener; import org.apache.spark.scheduler.SparkListenerJobStart; diff --git a/spark/spark3-shims/src/main/java/org/apache/zeppelin/spark/Spark3Shims.java b/spark/spark3-shims/src/main/java/org/apache/zeppelin/spark/Spark3Shims.java index 544bd0a67b..094fca62c1 100644 --- a/spark/spark3-shims/src/main/java/org/apache/zeppelin/spark/Spark3Shims.java +++ b/spark/spark3-shims/src/main/java/org/apache/zeppelin/spark/Spark3Shims.java @@ -18,7 +18,7 @@ package org.apache.zeppelin.spark; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.spark.SparkContext; import org.apache.spark.scheduler.SparkListener; import org.apache.spark.scheduler.SparkListenerJobStart;
[zeppelin] branch master updated: [ZEPPELIN-5493] Remove prefix in jdbc interpreter (#4463)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 679b3fbe66 [ZEPPELIN-5493] Remove prefix in jdbc interpreter (#4463) 679b3fbe66 is described below commit 679b3fbe666f0b7533f2bf8bea0d7dcfbfc2ac49 Author: Guanhua Li AuthorDate: Fri Oct 28 16:05:11 2022 +0800 [ZEPPELIN-5493] Remove prefix in jdbc interpreter (#4463) * Remove prefix in jdbc interpreter * update docs --- docs/interpreter/hive.md | 45 +--- docs/interpreter/jdbc.md | 8 +- .../org/apache/zeppelin/jdbc/JDBCInterpreter.java | 127 ++--- .../zeppelin/jdbc/JDBCUserConfigurations.java | 63 +- .../apache/zeppelin/jdbc/JDBCInterpreterTest.java | 120 --- 5 files changed, 122 insertions(+), 241 deletions(-) diff --git a/docs/interpreter/hive.md b/docs/interpreter/hive.md index 467e494857..94e49ce6a4 100644 --- a/docs/interpreter/hive.md +++ b/docs/interpreter/hive.md @@ -25,7 +25,7 @@ limitations under the License. ## Important Notice -Hive Interpreter will be deprecated and merged into JDBC Interpreter. +Hive Interpreter has been deprecated and merged into JDBC Interpreter. You can use Hive Interpreter by using JDBC Interpreter with same functionality. See the example below of settings and dependencies. @@ -36,19 +36,19 @@ See the example below of settings and dependencies. Value -hive.driver +default.driver org.apache.hive.jdbc.HiveDriver -hive.url +default.url jdbc:hive2://localhost:1 -hive.user +default.user hiveUser -hive.password +default.password hivePassword @@ -102,31 +102,6 @@ See the example below of settings and dependencies. ( Optional ) Other properties used by the driver - -${prefix}.driver - -Driver class path of %hive(${prefix}) - - -${prefix}.url - -Url of %hive(${prefix}) - - -${prefix}.user - -( Optional ) Username of the connection of %hive(${prefix}) - - -${prefix}.password - -( Optional ) Password of the connection of %hive(${prefix}) - - -${prefix}.xxx - -( Optional ) Other properties used by the driver of %hive(${prefix}) - zeppelin.jdbc.hive.timeout.threshold 6 @@ -144,8 +119,6 @@ See the example below of settings and dependencies. -This interpreter provides multiple configuration with `${prefix}`. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`. - ## Overview The [Apache Hive](https://hive.apache.org/) ™ data warehouse software facilitates querying and managing large datasets @@ -162,14 +135,6 @@ Basically, you can use select * from my_table; ``` -or - -```sql -%hive(etl) --- 'etl' is a ${prefix} -select * from my_table; -``` - You can also run multiple queries up to 10 by default. Changing these settings is not implemented yet. ### Apply Zeppelin Dynamic Forms diff --git a/docs/interpreter/jdbc.md b/docs/interpreter/jdbc.md index a94f7bde4d..62d62039eb 100644 --- a/docs/interpreter/jdbc.md +++ b/docs/interpreter/jdbc.md @@ -722,16 +722,16 @@ See [User Impersonation in interpreter](../usage/interpreter/user_impersonation. Value -hive.driver +default.driver org.apache.hive.jdbc.HiveDriver -hive.url +default.url jdbc:hive2://hive-server-host:2181/;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2 -hive.proxy.user.property -hive.server2.proxy.user +default.proxy.user.property +default.server2.proxy.user zeppelin.jdbc.auth.type diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index 705ad19192..702beb0795 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -326,7 +326,7 @@ public class JDBCInterpreter extends KerberosInterpreter { private void initConnectionPoolMap() { for (String key : jdbcUserConfigurationsMap.keySet()) { try { -closeDBPool(key, DEFAULT_KEY); +closeDBPool(key); } catch (SQLException e) { LOGGER.error("Error while closing database pool.", e); } @@ -356,10 +356,9 @@ public class JDBCInterpreter extends KerberosInterpreter { */ private String getUser(InterpreterContext context) { String user = context.getAuthenticationInfo().getUser(); -String dbPrefix = getDBPrefix(context); -if ("anonymous".equalsIgnoreCase(user) && basePropertiesMap.containsKey(dbPrefix)) {
[zeppelin] branch gh-pages updated: Zeppelin Security page (#4478)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch gh-pages in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/gh-pages by this push: new e1e50b28c6 Zeppelin Security page (#4478) e1e50b28c6 is described below commit e1e50b28c6125b94d01b122b471f5a18cff30d35 Author: Arnout Engelen AuthorDate: Wed Oct 19 10:09:07 2022 +0200 Zeppelin Security page (#4478) --- _includes/themes/zeppelin/_navigation.html | 1 + security.md| 76 ++ 2 files changed, 77 insertions(+) diff --git a/_includes/themes/zeppelin/_navigation.html b/_includes/themes/zeppelin/_navigation.html index a77c5d6e94..e0510247f2 100644 --- a/_includes/themes/zeppelin/_navigation.html +++ b/_includes/themes/zeppelin/_navigation.html @@ -33,6 +33,7 @@ 0.9.0 0.8.2 Older Versions +Security diff --git a/security.md b/security.md new file mode 100644 index 00..912b08e354 --- /dev/null +++ b/security.md @@ -0,0 +1,76 @@ +# Zeppelin Security + +This page explains what security characteristics can be expected from +Zeppelin, what measures operators of a Zeppelin instance will have to +take, and how to report any security issues found in the Zeppelin +software. + +## Code execution on the server + +It is the nature of the Zeppelin software that it allows +uploading code from the browser and executing it on the server. + +Because of this, you should make sure your Zeppelin instance is only +available to trusted users, and the server on which Zeppelin is +installed does not contain any secrets or have privileges beyond +those the users are trusted with. + +### Zeppelin on Docker + +An exception to the above is when the Zeppelin interpreter +is [run in a Docker container](https://zeppelin.apache.org/docs/latest/quickstart/docker.html). +This isolates the operating environment of the interpreter through the docker container. + +### Zeppelin on Kubernetes + +A similar exception exists when Zeppelin is +[deployed on Kubernetes](https://zeppelin.apache.org/docs/latest/quickstart/kubernetes.html). +In this case Zeppelin creates pods for individual interpreters, +and also the Spark interpreter is auto configured to use Spark +on Kubernetes in client mode. + +## JavaScript code execution in the browser + +Zeppelin allows notes to produce rich output, including HTML and even +executing JavaScript code. This means that when users view each others' +notes, HTML and JavaScript controlled by the creator of the note will +be executed in the browser that views it. + +Because of this, you should make sure your Zeppelin instance is only +available to trusted users. When deploying Zeppelin on a domain that +is shared with other applications, appropriate measures may have to be +taken to avoid a compromised Zeppelin notebook to also grant access +to other services on the same domain. + +## Authentication + +If you expose your Zeppelin instance on a network you don't fully trust, +you should configure [Apache Shiro authentication](https://zeppelin.apache.org/docs/latest/setup/security/shiro_authentication.html). + +Non-authenticated users cannot view, store or execute notes, so they +cannot execute code on the server or on other users' browsers. +Authenticated users, however, have the same access as described above, +so even when using authentication it is still important to only give +trusted users access to Zeppelin. Specifically, unless Docker or K8s +isolation has been configured as mentioned above, users technically +have access to all notes by other users. + +# Reporting security issues + +If you have found a potential security issue in Zeppelin, +such as a way to bypass the Shiro authentication, +we encourage you to report this problem at +[secur...@zeppelin.apache.org](mailto:secur...@zeppelin.apache.org). +This is a private mailing list. Please send one plain-text email +for each vulnerability you are reporting. + +## Vulnerability handling + +An overview of the vulnerability handling process is: + +* The reporter reports the vulnerability privately to [secur...@zeppelin.apache.org](mailto:secur...@zeppelin.apache.org). +* The Zeppelin project security team works privately with the reporter to resolve the vulnerability. +* The Zeppelin project creates a new release of the package the vulnerabilty affects to deliver its fix. +* The Zeppelin project publicly announces the vulnerability and describes how to apply the fix. + +Committers should read a [more detailed description of the process](https://www.apache.org/security/committers.html). Reporters of security vulnerabilities may also find it useful.
[zeppelin] branch master updated: [ZEPPELIN-5694] Support apache kyuubi (#4392)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 46ae9156ea [ZEPPELIN-5694] Support apache kyuubi (#4392) 46ae9156ea is described below commit 46ae9156ea14ea1a697d52407a677eedaa038ef5 Author: Guanhua Li AuthorDate: Sat Sep 24 11:16:01 2022 +0800 [ZEPPELIN-5694] Support apache kyuubi (#4392) * [ZEPPELIN-5694] Support apache kyuubi * SUPPORT KERBEROS MODE --- docs/interpreter/jdbc.md | 100 - docs/quickstart/sql_with_zeppelin.md | 1 + .../org/apache/zeppelin/jdbc/JDBCInterpreter.java | 5 +- 3 files changed, 104 insertions(+), 2 deletions(-) diff --git a/docs/interpreter/jdbc.md b/docs/interpreter/jdbc.md index c21c661bbc..a94f7bde4d 100644 --- a/docs/interpreter/jdbc.md +++ b/docs/interpreter/jdbc.md @@ -162,7 +162,7 @@ The last step is **Dependency Setting**. Since Zeppelin only includes `PostgreSQ -That's it. You can find more JDBC connection setting examples([Mysql](#mysql), [MariaDB](#mariadb), [Redshift](#redshift), [Apache Hive](#apache-hive), [Presto/Trino](#prestotrino), [Impala](#impala), [Apache Phoenix](#apache-phoenix), and [Apache Tajo](#apache-tajo)) in [this section](#examples). +That's it. You can find more JDBC connection setting examples([Mysql](#mysql), [MariaDB](#mariadb), [Redshift](#redshift), [Apache Hive](#apache-hive), [Presto/Trino](#prestotrino), [Impala](#impala), [Apache Kyuubi (Incubating)](#apache-kyuubi-(incubating)), [Apache Phoenix](#apache-phoenix), and [Apache Tajo](#apache-tajo)) in [this section](#examples). ## JDBC Interpreter Datasource Pool Configuration The Jdbc interpreter uses the connection pool technology, and supports users to do some personal configuration of the connection pool. For example, we can configure `default.validationQuery='select 1'` and `default.testOnBorrow=true` in the Interpreter configuration to avoid the "Invalid SessionHandle" runtime error caused by Session timeout when connecting to HiveServer2 through JDBC interpreter. @@ -827,6 +827,104 @@ Dependencies +### Apache Kyuubi (Incubating) + +Zeppelin connect to `Kyuubi` to run sql via `KyuubiHiveDriver`. There are 2 cases of connecting with Kyuubi: + +* Connect to Kyuubi without KERBEROS +* Connect to Kyuubi with KERBEROS + +Each case requires different settings. + +# Connect to Kyuubi without KERBEROS + +In this scenario, you need to make the following settings at least. Kyuubi engine run as user of `default.user`. + +Properties + + + +Name +Value + + +default.driver +org.apache.kyuubi.jdbc.KyuubiHiveDriver + + +default.url +jdbc:hive2://kyuubi-server:10009 + + + +Dependencies + + + +Artifact +Excludes + + +org.apache.kyuubi:kyuubi-hive-jdbc-shaded:1.5.2-incubating + + + +org.apache.hive:hive-jdbc:3.1.2 + + + + + +# Connect to Kyuubi with KERBEROS + +In this scenario, you need to make the following settings at least. Kyuubi engine run as user of client principal (`zeppelin.jdbc.principal`). + +Properties + + + +Name +Value + + +default.driver +org.apache.kyuubi.jdbc.KyuubiHiveDriver + + +default.url + jdbc:hive2://kyuubi-server:10009/default;principal={kyuubi_principal} + + +zeppelin.jdbc.auth.type +KERBEROS + + +zeppelin.jdbc.keytab.location +keytab of client + + +zeppelin.jdbc.principal +principal of client + + + +Dependencies + + + +Artifact +Excludes + + +org.apache.kyuubi:kyuubi-hive-jdbc-shaded:1.5.2-incubating + + + +org.apache.hive:hive-jdbc:3.1.2 + + + + ### Apache Phoenix Phoenix supports `thick` and `thin` connection types: diff --git a/docs/quickstart/sql_with_zeppelin.md b/docs/quickstart/sql_with_zeppelin.md index 2f04478020..b2c3beb651 100644 --- a/docs/quickstart/sql_with_zeppelin.md +++ b/docs/quickstart/sql_with_zeppelin.md @@ -35,6 +35,7 @@ The following guides explain how to use Apache Zeppelin that enables you to writ * [Apache Hive](../interpreter/jdbc.html#apache-hive) * [Presto/Trino](../interpreter/jdbc.html#prestotrino) * [Impala](../interpreter/jdbc.html#impala) + * [Apache Kyuubi (Incubating)](../interpreter/jdbc.html#apache-kyuubi-incubating) * [Apache Phoenix](../interpreter/jdbc.html#apache-phoenix) * [Apache Drill](../interpreter/jdbc.html#apache-drill) * [Apache Tajo](../interpreter/jdbc.html#apache-tajo) diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index f5302dc48a..705ad19192 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zep
[zeppelin] branch master updated: [ZEPPELIN-5761] Support flink 1.15.1 (#4404)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 65dcdb8148 [ZEPPELIN-5761] Support flink 1.15.1 (#4404) 65dcdb8148 is described below commit 65dcdb814851eeedd4322bbb8cbfc251af9b44a2 Author: Jeff Zhang AuthorDate: Tue Jul 19 11:07:12 2022 +0800 [ZEPPELIN-5761] Support flink 1.15.1 (#4404) --- flink/pom.xml | 2 +- testing/env_python_3_with_flink_115.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flink/pom.xml b/flink/pom.xml index dc1664a0cf..d432293e2f 100644 --- a/flink/pom.xml +++ b/flink/pom.xml @@ -46,7 +46,7 @@ 1.12.4 1.13.2 1.14.0 -1.15.0 +1.15.1 2.11.12 2.11 diff --git a/testing/env_python_3_with_flink_115.yml b/testing/env_python_3_with_flink_115.yml index a434029e4e..f62522fe2c 100644 --- a/testing/env_python_3_with_flink_115.yml +++ b/testing/env_python_3_with_flink_115.yml @@ -25,5 +25,5 @@ dependencies: - jinja2=3.0.3 - pip - pip: - - apache-flink==1.15.0 + - apache-flink==1.15.1
[zeppelin] branch master updated: [ZEPPELIN-5755] Support Spark 3.3 (#4388)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new ca216a512e [ZEPPELIN-5755] Support Spark 3.3 (#4388) ca216a512e is described below commit ca216a512e5c6ad474ddece5ecc17cfe594de6bc Author: Jeff Zhang AuthorDate: Tue Jul 12 08:49:35 2022 +0800 [ZEPPELIN-5755] Support Spark 3.3 (#4388) * [ZEPPELIN-5755] Support Spark 3.3 * use hadoop3 profile for spark 3.3 --- .github/workflows/core.yml | 10 +++- pom.xml| 2 +- spark/interpreter/pom.xml | 14 +- .../apache/zeppelin/spark/IPySparkInterpreter.java | 5 ++ .../apache/zeppelin/spark/PySparkInterpreter.java | 6 +++ .../src/main/resources/python/zeppelin_ipyspark.py | 6 ++- .../src/main/resources/python/zeppelin_pyspark.py | 8 ++-- .../zeppelin/spark/SparkSqlInterpreterTest.java| 8 ++-- spark/scala-2.13/pom.xml | 2 +- .../org/apache/zeppelin/spark/SparkVersion.java| 6 ++- testing/env_python_3.7_with_R.yml | 2 +- testing/env_python_3.8_with_R.yml | 2 +- testing/env_python_3_with_R.yml| 2 +- testing/env_python_3_with_R_and_tensorflow.yml | 2 +- .../integration/SparkIntegrationTest33.java| 56 ++ .../integration/ZeppelinSparkClusterTest33.java| 40 16 files changed, 155 insertions(+), 16 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index b9b3953fc8..aceaf9fa11 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -325,7 +325,7 @@ jobs: run: | R -e "IRkernel::installspec()" - name: run tests on hadoop${{ matrix.hadoop }} -run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration -Phadoop${{ matrix.hadoop }} -Pintegration -B -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest30,SparkIntegrationTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest31,ZeppelinSparkClusterTest32,SparkIntegrationTest32 -DfailIfNoTests=false +run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration -Phadoop${{ matrix.hadoop }} -Pintegration -B -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest30,SparkIntegrationTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest31,ZeppelinSparkClusterTest32,SparkIntegrationTest32,ZeppelinSparkClusterTest33,SparkIntegrationTest33 -DfailIfNoTests=false # test on spark for each spark version & scala version spark-test: @@ -395,6 +395,14 @@ jobs: run: | rm -rf spark/interpreter/metastore_db ./mvnw test -DskipRat -pl spark-submit,spark/interpreter -Pspark-3.2 -Pspark-scala-2.13 -Phadoop2 -Pintegration -B -DfailIfNoTests=false + - name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python }} +run: | + rm -rf spark/interpreter/metastore_db + ./mvnw test -DskipRat -pl spark-submit,spark/interpreter -Pspark-3.3 -Pspark-scala-2.12 -Phadoop3 -Pintegration -B -DfailIfNoTests=false + - name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python }} +run: | + rm -rf spark/interpreter/metastore_db + ./mvnw test -DskipRat -pl spark-submit,spark/interpreter -Pspark-3.3 -Pspark-scala-2.13 -Phadoop3 -Pintegration -B -DfailIfNoTests=false livy-0-5-with-spark-2-2-0-under-python3: runs-on: ubuntu-20.04 diff --git a/pom.xml b/pom.xml index 668bb99d70..78b011339c 100644 --- a/pom.xml +++ b/pom.xml @@ -131,7 +131,7 @@ 4.5.13 4.0.2 1.21 -3.10 +3.12.0 1.8 2.7 1.3 diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index e539f99732..8f154ba609 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -572,10 +572,22 @@ - spark-3.2 + spark-3.3 true + +4.1.17 +4.1.19 +4.2.4 +3.3.0 +2.5.0 +0.10.9.5 + + + + + spark-3.2 4.1.17 4.1.19 diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java index ab6b3dbf29..2e945ed2bb 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java @@ -161,6 +161,11 @@ public class IPySparkInterpreter extends IPythonInterpreter { return sparkInterpreter.getSparkVersion().getMajorVersion() == 3; } + // Used by PyS
[zeppelin] branch master updated: [ZEPPELIN-5600] Support Flink 1.15 (#4335)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new deaf3d3e9f [ZEPPELIN-5600] Support Flink 1.15 (#4335) deaf3d3e9f is described below commit deaf3d3e9faf125e6d255c6cb405e650293b8b95 Author: Jeff Zhang AuthorDate: Fri May 13 14:01:55 2022 +0800 [ZEPPELIN-5600] Support Flink 1.15 (#4335) --- .github/workflows/core.yml | 27 +- docs/interpreter/flink.md | 7 + flink/flink-scala-parent/pom.xml | 40 +- .../org/apache/zeppelin/flink/TableEnvFactory.java | 138 + .../internal/ScalaShellStreamEnvironment.java | 21 +- .../zeppelin/flink/sql/AbstractStreamSqlJob.java | 16 +- .../zeppelin/flink/FlinkScalaInterpreter.scala | 17 +- .../zeppelin/flink/internal/FlinkILoop.scala | 2 +- .../java/org/apache/zeppelin/flink/FlinkShims.java | 11 +- .../org/apache/zeppelin/flink/Flink112Shims.java | 19 +- .../org/apache/zeppelin/flink/Flink113Shims.java | 19 +- .../org/apache/zeppelin/flink/Flink114Shims.java | 18 +- flink/flink1.15-shims/pom.xml | 200 +++ .../org/apache/zeppelin/flink/Flink115Shims.java} | 65 ++- .../zeppelin/flink/Flink115SqlInterpreter.java | 590 + .../java/org/apache/zeppelin/flink/PrintUtils.java | 318 +++ .../zeppelin/flink/TimestampStringUtils.java | 143 + .../flink/shims115/CollectStreamTableSink.java | 97 flink/pom.xml | 44 +- testing/env_python_3_with_flink_115.yml| 29 + .../integration/ZeppelinFlinkClusterTest115.java | 38 ++ .../launcher/FlinkInterpreterLauncher.java | 21 +- 22 files changed, 1666 insertions(+), 214 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 30b675b6e9..b9b3953fc8 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -198,7 +198,7 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment run: | - ./mvnw install -DskipTests -DskipRat -Phadoop2 -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.11,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.11,flink/flink-scala-2.12,jdbc,shell -am + ./mvnw install -DskipTests -DskipRat -Phadoop2 -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.11,spark/scala-2.12,spark/scala-2.13,markdown,flink-cmd,flink/flink-scala-2.11,flink/flink-scala-2.12,jdbc,shell -am -Pflink-114 ./mvnw package -DskipRat -pl zeppelin-plugins -amd -DskipTests -B - name: Setup conda environment with python 3.7 and R uses: conda-incubator/setup-miniconda@v2 @@ -216,13 +216,16 @@ jobs: R -e "IRkernel::installspec()" - name: run tests run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration -Phadoop2 -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest,ShellIntegrationTest,JdbcIntegrationTest - + - name: Print zeppelin logs +if: always() +run: if [ -d "logs" ]; then cat logs/*; fi + flink-test-and-flink-integration-test: runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: -flink: [112, 113, 114] +flink: [112, 113, 114, 115] steps: - name: Checkout uses: actions/checkout@v2 @@ -244,10 +247,16 @@ jobs: key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-zeppelin- - - name: install environment + - name: install environment for flink before 1.15 (exclusive) +if: matrix.flink != '115' run: | ./mvnw install -DskipTests -DskipRat -am -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration -B ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests -B + - name: install environment for flink after 1.15 (inclusive) +if: matrix.flink == '115' +run: | + ./mvnw install -DskipTests -DskipRat -am -pl flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Phadoop2 -Pintegration -B + ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests -B - name: Setup conda environment with python 3.7 and uses: conda-incubator/setup-miniconda@v2 with: @@ -259,8 +268,16 @@ jobs: channel-priority: true auto-activate-base: false use-mamba: true - - name: run tests + - name: run tests f
[zeppelin] branch master updated: The title `Zeppelin on Yarn` is somewhat misleading. It should be consistent with the other title `Zeppelin Interpreter on Docker`. (#4364)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 8e52181280 The title `Zeppelin on Yarn` is somewhat misleading. It should be consistent with the other title `Zeppelin Interpreter on Docker`. (#4364) 8e52181280 is described below commit 8e521812809a14603b0d4ae6917ca2e245844e93 Author: Guanhua Li AuthorDate: Sun May 8 22:37:25 2022 +0800 The title `Zeppelin on Yarn` is somewhat misleading. It should be consistent with the other title `Zeppelin Interpreter on Docker`. (#4364) Also fix a small syntax problem. --- docs/quickstart/docker.md | 4 ++-- docs/quickstart/yarn.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/quickstart/docker.md b/docs/quickstart/docker.md index 0c6a478ff1..5e38547692 100644 --- a/docs/quickstart/docker.md +++ b/docs/quickstart/docker.md @@ -19,9 +19,9 @@ limitations under the License. --> {% include JB/setup %} -# Zeppelin interpreter on Docker +# Zeppelin Interpreter on Docker -Zeppelin service runs on local server. zeppelin is able to run the interpreter in the docker container, Isolating the operating environment of the interpreter through the docker container. Zeppelin can be easily used without having to install python, spark, etc. on the local node. +Zeppelin service runs on local server. Zeppelin is able to run the interpreter in the docker container, Isolating the operating environment of the interpreter through the docker container. Zeppelin can be easily used without having to install python, spark, etc. on the local node. Key benefits are diff --git a/docs/quickstart/yarn.md b/docs/quickstart/yarn.md index 0aee85c884..19808a26b4 100644 --- a/docs/quickstart/yarn.md +++ b/docs/quickstart/yarn.md @@ -19,11 +19,11 @@ limitations under the License. --> {% include JB/setup %} -# Zeppelin on Yarn +# Zeppelin Interpreter on Yarn -Zeppelin on yarn means to run interpreter process in yarn container. The key benefit is the scalability, you won't run out of memory +Zeppelin is able to run interpreter process in yarn container. The key benefit is the scalability, you won't run out of memory of the zeppelin server host if you run large amount of interpreter processes. ## Prerequisites
[zeppelin] branch master updated: [ZEPPELIN-5729] Increase default value of zeppelin.interpreter.connect.timeout (#4366)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new b2e052a653 [ZEPPELIN-5729] Increase default value of zeppelin.interpreter.connect.timeout (#4366) b2e052a653 is described below commit b2e052a653459f0de7dfe33feaa796233439316e Author: Guanhua Li AuthorDate: Sun May 8 15:31:31 2022 +0800 [ZEPPELIN-5729] Increase default value of zeppelin.interpreter.connect.timeout (#4366) update and fix docs --- conf/zeppelin-site.xml.template | 2 +- docs/setup/operation/configuration.md | 4 ++-- .../src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template index 5b96312f79..dab1204387 100755 --- a/conf/zeppelin-site.xml.template +++ b/conf/zeppelin-site.xml.template @@ -417,7 +417,7 @@ zeppelin.interpreter.connect.timeout - 6 + 60 Interpreter process connect timeout in msec. diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md index 026581d57e..6527974834 100644 --- a/docs/setup/operation/configuration.md +++ b/docs/setup/operation/configuration.md @@ -340,8 +340,8 @@ Sources descending by priority: ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT zeppelin.interpreter.connect.timeout -3 -Output message from interpreter exceeding the limit will be truncated +60 +Interpreter process connect timeout in msec. ZEPPELIN_DEP_LOCALREPO diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 96a4b10de8..985a82d9f7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -936,7 +936,7 @@ public class ZeppelinConfiguration { ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"), ZEPPELIN_INTERPRETER_DEP_MVNREPO("zeppelin.interpreter.dep.mvnRepo", "https://repo1.maven.org/maven2/;), - ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 6), + ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 60), ZEPPELIN_INTERPRETER_CONNECTION_POOL_SIZE("zeppelin.interpreter.connection.poolsize", 100), ZEPPELIN_INTERPRETER_GROUP_DEFAULT("zeppelin.interpreter.group.default", "spark"), ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit", 1024 * 100),
[zeppelin] branch master updated: [ZEPPELIN-5614] A JDBC interpreter property for disable setting tag for application of hive engines (#4360)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 93126f2504 [ZEPPELIN-5614] A JDBC interpreter property for disable setting tag for application of hive engines (#4360) 93126f2504 is described below commit 93126f25049776009dad5c6086f13d25e2fcbf72 Author: Guanhua Li AuthorDate: Wed Apr 27 13:43:49 2022 +0800 [ZEPPELIN-5614] A JDBC interpreter property for disable setting tag for application of hive engines (#4360) * [ZEPPELIN-5614] Introduce a JDBC interpreter property for hive engines which don't need to set yarn application tag * update for review * update docs * add default value when property is not set. --- docs/interpreter/hive.md | 15 +++ docs/interpreter/jdbc.md | 2 ++ .../java/org/apache/zeppelin/jdbc/JDBCInterpreter.java| 4 jdbc/src/main/resources/interpreter-setting.json | 9 - 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/docs/interpreter/hive.md b/docs/interpreter/hive.md index 86602fcc27..467e494857 100644 --- a/docs/interpreter/hive.md +++ b/docs/interpreter/hive.md @@ -127,6 +127,21 @@ See the example below of settings and dependencies. ( Optional ) Other properties used by the driver of %hive(${prefix}) + +zeppelin.jdbc.hive.timeout.threshold +6 +Timeout for hive job timeout + + +zeppelin.jdbc.hive.monitor.query_interval +1000 +Query interval for hive statement + + +zeppelin.jdbc.hive.engines.tag.enable +true +Set application tag for applications started by hive engines + This interpreter provides multiple configuration with `${prefix}`. User can set a multiple connection properties by this prefix. It can be used like `%hive(${prefix})`. diff --git a/docs/interpreter/jdbc.md b/docs/interpreter/jdbc.md index 4824edc4b4..c21c661bbc 100644 --- a/docs/interpreter/jdbc.md +++ b/docs/interpreter/jdbc.md @@ -739,6 +739,8 @@ See [User Impersonation in interpreter](../usage/interpreter/user_impersonation. +See [Hive Interpreter](../interpreter/hive.html) for more properties about Hive interpreter. + ### Presto/Trino Properties diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index 58068e2174..f5302dc48a 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -604,6 +604,10 @@ public class JDBCInterpreter extends KerberosInterpreter { // only add tags for hive jdbc private String appendTagsToURL(String url, InterpreterContext context) { +if (!Boolean.parseBoolean(getProperty("zeppelin.jdbc.hive.engines.tag.enable", "true"))) { + return url; +} + StringBuilder builder = new StringBuilder(url); if (url.startsWith("jdbc:hive2:")) { Integer lastIndexOfQMark = builder.indexOf("?"); diff --git a/jdbc/src/main/resources/interpreter-setting.json b/jdbc/src/main/resources/interpreter-setting.json index 1b2b4d49ea..a723660d6e 100644 --- a/jdbc/src/main/resources/interpreter-setting.json +++ b/jdbc/src/main/resources/interpreter-setting.json @@ -77,7 +77,7 @@ "zeppelin.jdbc.auth.kerberos.proxy.enable": { "envName": null, "propertyName": "zeppelin.jdbc.auth.kerberos.proxy.enable", -"defaultValue": "true", +"defaultValue": true, "description": "When auth type is Kerberos, enable/disable Kerberos proxy with the login user to get the connection. Default value is true.", "type": "checkbox" }, @@ -143,6 +143,13 @@ "defaultValue": "1000", "description": "Query interval for hive statement", "type": "number" + }, + "zeppelin.jdbc.hive.engines.tag.enable": { +"envName": null, +"propertyName": "zeppelin.jdbc.hive.engines.tag.enable", +"defaultValue": true, +"description": "Set application tag for applications started by hive engines", +"type": "checkbox" } }, "editor": {
[zeppelin] branch master updated: [ZEPPELIN-5728] Absence of cancelNote method in Zeppelin Client Api (#4362)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 8672d6ca23 [ZEPPELIN-5728] Absence of cancelNote method in Zeppelin Client Api (#4362) 8672d6ca23 is described below commit 8672d6ca23f174179b2525f5fb6b550328418fc7 Author: Guanhua Li AuthorDate: Wed Apr 27 13:42:44 2022 +0800 [ZEPPELIN-5728] Absence of cancelNote method in Zeppelin Client Api (#4362) * [ZEPPELIN-5728] Absence of cancelNote method in Zeppelin Client Api * add test --- .../client/examples/ZeppelinClientExample.java | 2 + .../org/apache/zeppelin/client/ZeppelinClient.java | 15 .../apache/zeppelin/rest/NotebookRestApiTest.java | 44 ++ 3 files changed, 61 insertions(+) diff --git a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample.java b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample.java index 7254568912..6e6e043c52 100644 --- a/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample.java +++ b/zeppelin-client-examples/src/main/java/org/apache/zeppelin/client/examples/ZeppelinClientExample.java @@ -56,6 +56,8 @@ public class ZeppelinClientExample { "%python\nimport time\ntime.sleep(5)\nprint('done')"); zClient.submitParagraph(noteId, paragraphId2); zClient.waitUtilParagraphRunning(noteId, paragraphId2); + // It's also ok here to call zClient.cancelNote(noteId); + // CancelNote() would cancel all paragraphs in the note. zClient.cancelParagraph(noteId, paragraphId2); paragraphResult = zClient.waitUtilParagraphFinish(noteId, paragraphId2); System.out.println("Added new paragraph, submit it then cancel it"); diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java index 2f8f6bc527..bec349e0bc 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java @@ -502,6 +502,21 @@ public class ZeppelinClient { return queryNoteResult(noteId); } + /** + * Cancel a running note. + * + * @param noteId + * @throws Exception + */ + public void cancelNote(String noteId) throws Exception { +HttpResponse response = Unirest +.delete("/notebook/job/{noteId}") +.routeParam("noteId", noteId) +.asJson(); +checkResponse(response); +JsonNode jsonNode = response.getBody(); +checkJsonNodeStatus(jsonNode); + } /** * Import note with given note json content to the specified notePath. diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java index 4744832847..95c167d1b5 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/NotebookRestApiTest.java @@ -430,6 +430,50 @@ public class NotebookRestApiTest extends AbstractTestRestApi { } } + @Test + public void testCancelNoteJob() throws Exception { +LOG.info("Running testCancelNoteJob"); +String note1Id = null; +Notebook notebook = TestUtils.getInstance(Notebook.class); +try { + note1Id = notebook.createNote("note1", anonymous); + // Add 3 paragraphs for the note. + List paragraphs = notebook.processNote(note1Id, + note1 -> { +List paragraphsList = new ArrayList<>(); +for (int i = 0; i < 3; i++) { + Paragraph p1 = note1.addNewParagraph(AuthenticationInfo.ANONYMOUS); + p1.setText("%python\nimport time\ntime.sleep(10)\nprint('done')"); + note1.run(p1.getId()); + paragraphsList.add(p1); +} +return paragraphsList; + }); + //The first paragraph is running, and the other two is pending. + paragraphs.get(0).waitUntilRunning(); + + + // cancel running note + CloseableHttpResponse delete = httpDelete("/notebook/job/" + note1Id); + assertThat(delete, isAllowed()); + Map resp = gson.fromJson(EntityUtils.toString(delete.getEntity(), StandardCharsets.UTF_8), + new TypeToken>() { + }.getType()); + assertEquals("OK", resp.get("status")); + delete.close(); + for (Paragraph p : paragraphs) { +p.waitUntilFinished(); +assertEquals(Job.Status.ABORT, p.getStatus()
[zeppelin] branch master updated: [HOTFIX] Fix Python plotting test failure
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new a6769cbddf [HOTFIX] Fix Python plotting test failure a6769cbddf is described below commit a6769cbddf2ed5d6ca0e9f5f95c7641a4b605c99 Author: Jeff Zhang AuthorDate: Fri Apr 22 15:35:16 2022 +0800 [HOTFIX] Fix Python plotting test failure --- .../org/apache/zeppelin/python/IPythonInterpreterTest.java | 10 +- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index 65e3786139..6f44cbb4a4 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -307,16 +307,8 @@ public class IPythonInterpreterTest extends BasePythonInterpreterTest { "df.hvplot()", context); assertEquals(context.out.toInterpreterResultMessage().get(0).getData(), InterpreterResult.Code.SUCCESS, result.code()); -interpreterResultMessages = context.out.toInterpreterResultMessage(); - -assertEquals(interpreterResultMessages.size() + ":" + context.out.toString(), -3, interpreterResultMessages.size()); -// the first message is the warning text message. -assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(0).getType()); -assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(1).getType()); -assertEquals(InterpreterResult.Type.HTML, interpreterResultMessages.get(2).getType()); // docs_json is the source data of plotting which bokeh would use to render the plotting. - assertTrue(interpreterResultMessages.get(2).getData().contains("docs_json")); +assertTrue(context.out.toString(), context.out.toString().contains("docs_json")); }
[zeppelin] branch master updated: [ZEPPELIN-5699] Trim space in ConfInterpreter
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 4b269f88fc [ZEPPELIN-5699] Trim space in ConfInterpreter 4b269f88fc is described below commit 4b269f88fc4a9d59a140b10260064a67382b6916 Author: Jeff Zhang AuthorDate: Wed Mar 30 11:33:19 2022 +0800 [ZEPPELIN-5699] Trim space in ConfInterpreter ### What is this PR for? Trim the space for the key & value of ConfInterpreter. Unit test is added. ### What type of PR is it? [Bug Fix ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5669 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4336 from zjffdu/ZEPPELIN-5699 and squashes the following commits: a4a471e1bf [Jeff Zhang] [ZEPPELIN-5699] Trim space in ConfInterpreter --- .../zeppelin/interpreter/ConfInterpreter.java | 4 +- .../zeppelin/interpreter/ConfInterpreterTest.java | 56 +- 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java index f343790862..9c5d02debb 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ConfInterpreter.java @@ -67,7 +67,9 @@ public class ConfInterpreter extends Interpreter { finalProperties.putAll(getProperties()); Properties newProperties = new Properties(); newProperties.load(new StringReader(st)); - finalProperties.putAll(newProperties); + for (String key : newProperties.stringPropertyNames()) { +finalProperties.put(key.trim(), newProperties.getProperty(key).trim()); + } LOGGER.debug("Properties for InterpreterGroup: {} is {}", interpreterGroupId, finalProperties); interpreterSetting.setInterpreterGroupProperties(interpreterGroupId, finalProperties); return new InterpreterResult(InterpreterResult.Code.SUCCESS); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ConfInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ConfInterpreterTest.java index 41f5d73b00..3a8717ff37 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ConfInterpreterTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/ConfInterpreterTest.java @@ -21,8 +21,9 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreter; import org.junit.Before; import org.junit.Test; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import java.util.Properties; + +import static org.junit.Assert.*; public class ConfInterpreterTest extends AbstractInterpreterTest { @@ -59,6 +60,57 @@ public class ConfInterpreterTest extends AbstractInterpreterTest { assertEquals(InterpreterResult.Code.ERROR, result.code); } + @Test + public void testPropertyTrim() throws InterpreterException { +assertTrue(interpreterFactory.getInterpreter("test.conf", executionContext) instanceof ConfInterpreter); +ConfInterpreter confInterpreter = (ConfInterpreter) interpreterFactory.getInterpreter("test.conf", executionContext); + +InterpreterContext context = InterpreterContext.builder() +.setNoteId("noteId") +.setParagraphId("paragraphId") +.build(); + +// space before key and space after values +InterpreterResult result = confInterpreter.interpret(" property_1 \tnew_value \n new_property \t dummy_value \n", context); +assertEquals(InterpreterResult.Code.SUCCESS, result.code); + +assertTrue(interpreterFactory.getInterpreter("test", executionContext) instanceof RemoteInterpreter); +RemoteInterpreter remoteInterpreter = (RemoteInterpreter) interpreterFactory.getInterpreter("test", executionContext); +remoteInterpreter.interpret("hello world", context); +Properties intpProperties = remoteInterpreter.getProperties(); +// Total 7 properties, +// 3 built-in properties (zeppelin.interpreter.output.limit, zeppelin.interpreter.localRepo, zeppelin.interpreter.connection.poolsize) +assertEquals(7, intpProperties.size()); + assertNotNull(intpProperties.getProperty("zeppelin.interpreter.output.limit")); + assertNotNull(intpP
[zeppelin] branch master updated: [ZEPPELIN-5027] upgrade jackson to avoid cve
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 70c791bae4 [ZEPPELIN-5027] upgrade jackson to avoid cve 70c791bae4 is described below commit 70c791bae4832a04442a3cdbaec98432f829949f Author: PJ Fanning AuthorDate: Mon Mar 28 20:35:10 2022 +0200 [ZEPPELIN-5027] upgrade jackson to avoid cve ### What is this PR for? [ZEPPELIN-5027] upgrade jackson to avoid cve ### What type of PR is it? Bug Fix ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN/ZEPPELIN-5027 ### How should this be tested? * Strongly recommended: add automated unit tests for any new or changed behavior * Outline any manual steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: PJ Fanning Closes #4333 from pjfanning/patch-3 and squashes the following commits: 201ca3515a [PJ Fanning] Update pom.xml 43cd25d732 [PJ Fanning] [ZEPPELIN-5027] upgrade jackson to avoid cve --- ksql/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ksql/pom.xml b/ksql/pom.xml index 6d7ba3b053..e9c178fd0f 100644 --- a/ksql/pom.xml +++ b/ksql/pom.xml @@ -40,13 +40,13 @@ com.fasterxml.jackson.core jackson-databind - 2.10.3 + 2.10.5.1 - org.apache.commons + commons-io commons-io - 1.3.2 + 2.11.0
[zeppelin] branch master updated: [HOTFIX] Fix Flink UT fails
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 50f42b8 [HOTFIX] Fix Flink UT fails 50f42b8 is described below commit 50f42b86cde8dc28ef5ecb5375304a4fad38a4a1 Author: Jeff Zhang AuthorDate: Thu Mar 31 15:54:08 2022 +0800 [HOTFIX] Fix Flink UT fails --- elasticsearch/pom.xml | 8 .../src/main/java/org/apache/zeppelin/spark/SparkShims.java | 4 ++-- zeppelin-interpreter-parent/pom.xml | 2 +- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/elasticsearch/pom.xml b/elasticsearch/pom.xml index d4392ac..e56843b 100644 --- a/elasticsearch/pom.xml +++ b/elasticsearch/pom.xml @@ -38,6 +38,8 @@ 18.0 0.1.6 1.4.9 + +1.3 @@ -81,6 +83,12 @@ ${unirest.version} + + org.hamcrest + hamcrest-all + ${hamcrest.all.version} + test + diff --git a/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkShims.java b/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkShims.java index 3fff0f0..709f484 100644 --- a/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkShims.java +++ b/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkShims.java @@ -17,7 +17,8 @@ package org.apache.zeppelin.spark; -import com.google.common.annotations.VisibleForTesting; + +import org.apache.zeppelin.shaded.com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.util.VersionInfo; import org.apache.hadoop.util.VersionUtil; import org.apache.zeppelin.interpreter.InterpreterContext; @@ -162,7 +163,6 @@ public abstract class SparkShims { || (VersionUtil.compareVersions(HADOOP_VERSION_3_0_0, version) <= 0); } - @VisibleForTesting public static void reset() { sparkShims = null; } diff --git a/zeppelin-interpreter-parent/pom.xml b/zeppelin-interpreter-parent/pom.xml index da1e438..82986d5 100644 --- a/zeppelin-interpreter-parent/pom.xml +++ b/zeppelin-interpreter-parent/pom.xml @@ -50,7 +50,7 @@ io.atomix - atomix + *
[zeppelin] branch master updated: [ZEPPELIN-5604] Interpreter scheduler may not be shutdown properly
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new f85d415 [ZEPPELIN-5604] Interpreter scheduler may not be shutdown properly f85d415 is described below commit f85d415225860afa8ff472189893dcd7370d3310 Author: Jeff Zhang AuthorDate: Tue Dec 7 23:56:50 2021 +0800 [ZEPPELIN-5604] Interpreter scheduler may not be shutdown properly ### What is this PR for? The issue will cause the thread used by the interpreter scheduler is not released, so overtime the thread pool may be exhausted. The root cause is that some uncaught exception may happen when closing interpreter, which causes the interpreter scheduler shutdown is not invoked. This PR put the scheduler shutdown in the finally block, so that it is always called. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5604 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4307 from zjffdu/ZEPPELIN-5604 and squashes the following commits: 02d9a61fec [Jeff Zhang] [ZEPPELIN-5604] Interpreter scheduler may not be shutdown properly --- .../interpreter/ManagedInterpreterGroup.java | 29 +++--- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java index fb17542..51aeab7 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/ManagedInterpreterGroup.java @@ -144,29 +144,28 @@ public class ManagedInterpreterGroup extends InterpreterGroup { private void closeInterpreter(Interpreter interpreter) { Scheduler scheduler = interpreter.getScheduler(); - -if (Boolean.parseBoolean( -interpreter.getProperty("zeppelin.interpreter.close.cancel_job", "true"))) { - for (final Job job : scheduler.getAllJobs()) { -if (!job.isTerminated()) { - job.abort(); - job.setStatus(Job.Status.ABORT); - LOGGER.info("Job {} aborted ", job.getJobName()); +try { + if (Boolean.parseBoolean( + interpreter.getProperty("zeppelin.interpreter.close.cancel_job", "true"))) { +for (final Job job : scheduler.getAllJobs()) { + if (!job.isTerminated()) { +job.abort(); +job.setStatus(Job.Status.ABORT); +LOGGER.info("Job {} aborted ", job.getJobName()); + } } + } else { +LOGGER.info("Keep job running while closing interpreter: {}", interpreter.getClassName()); } -} else { - LOGGER.info("Keep job running while closing interpreter: {}", interpreter.getClassName()); -} -try { LOGGER.info("Trying to close interpreter {}", interpreter.getClassName()); interpreter.close(); } catch (InterpreterException e) { LOGGER.warn("Fail to close interpreter {}", interpreter.getClassName(), e); +} finally { + //TODO(zjffdu) move the close of schedule to Interpreter + SchedulerFactory.singleton().removeScheduler(scheduler.getName()); } - -//TODO(zjffdu) move the close of schedule to Interpreter -SchedulerFactory.singleton().removeScheduler(scheduler.getName()); } public synchronized List getOrCreateSession(String user, String sessionId) {
[zeppelin] branch master updated: [ZEPPELIN-5669] Check pyflink folder existence in yarn application mode
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 148129e [ZEPPELIN-5669] Check pyflink folder existence in yarn application mode 148129e is described below commit 148129e4029009f1a6131144917262d2ec71d730 Author: Jeff Zhang AuthorDate: Thu Mar 17 10:34:19 2022 +0800 [ZEPPELIN-5669] Check pyflink folder existence in yarn application mode ### What is this PR for? Trivial PR to check the existence of the folder, and throw a meaningful error when it doesn't exist. ### What type of PR is it? [ Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5669 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4320 from zjffdu/ZEPPELIN-5669 and squashes the following commits: 769d0aa984 [Jeff Zhang] [ZEPPELIN-5669] Check pyflink folder existence in yarn application mode --- .../main/java/org/apache/zeppelin/flink/Flink112Shims.java | 12 .../main/java/org/apache/zeppelin/flink/Flink113Shims.java | 12 .../main/java/org/apache/zeppelin/flink/Flink114Shims.java | 12 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/flink/flink1.12-shims/src/main/java/org/apache/zeppelin/flink/Flink112Shims.java b/flink/flink1.12-shims/src/main/java/org/apache/zeppelin/flink/Flink112Shims.java index e182f97..757e7a4 100644 --- a/flink/flink1.12-shims/src/main/java/org/apache/zeppelin/flink/Flink112Shims.java +++ b/flink/flink1.12-shims/src/main/java/org/apache/zeppelin/flink/Flink112Shims.java @@ -125,20 +125,24 @@ public class Flink112Shims extends FlinkShims { if ("yarn-application".equalsIgnoreCase(mode)) { // for yarn application mode, FLINK_HOME is container working directory String flinkHome = new File(".").getAbsolutePath(); - return getPyFlinkPythonPath(flinkHome + "/lib/python"); + return getPyFlinkPythonPath(new File(flinkHome + "/lib/python")); } String flinkHome = System.getenv("FLINK_HOME"); if (StringUtils.isNotBlank(flinkHome)) { - return getPyFlinkPythonPath(flinkHome + "/opt/python"); + return getPyFlinkPythonPath(new File(flinkHome + "/opt/python")); } else { throw new IOException("No FLINK_HOME is specified"); } } - private String getPyFlinkPythonPath(String pyFlinkFolder) { + private String getPyFlinkPythonPath(File pyFlinkFolder) throws IOException { LOGGER.info("Getting pyflink lib from {}", pyFlinkFolder); -List depFiles = Arrays.asList(new File(pyFlinkFolder).listFiles()); +if (!pyFlinkFolder.exists() || !pyFlinkFolder.isDirectory()) { + throw new IOException(String.format("PyFlink folder %s does not exist or is not a folder", + pyFlinkFolder.getAbsolutePath())); +} +List depFiles = Arrays.asList(pyFlinkFolder.listFiles()); StringBuilder builder = new StringBuilder(); for (File file : depFiles) { LOGGER.info("Adding extracted file {} to PYTHONPATH", file.getAbsolutePath()); diff --git a/flink/flink1.13-shims/src/main/java/org/apache/zeppelin/flink/Flink113Shims.java b/flink/flink1.13-shims/src/main/java/org/apache/zeppelin/flink/Flink113Shims.java index a4743e8..792174a 100644 --- a/flink/flink1.13-shims/src/main/java/org/apache/zeppelin/flink/Flink113Shims.java +++ b/flink/flink1.13-shims/src/main/java/org/apache/zeppelin/flink/Flink113Shims.java @@ -128,20 +128,24 @@ public class Flink113Shims extends FlinkShims { if ("yarn-application".equalsIgnoreCase(mode)) { // for yarn application mode, FLINK_HOME is container working directory String flinkHome = new File(".").getAbsolutePath(); - return getPyFlinkPythonPath(flinkHome + "/lib/python"); + return getPyFlinkPythonPath(new File(flinkHome + "/lib/python")); } String flinkHome = System.getenv("FLINK_HOME"); if (StringUtils.isNotBlank(flinkHome)) { - return getPyFlinkPythonPath(flinkHome + "/opt/python"); + return getPyFlinkPythonPath(new File(flinkHome + "/opt/python")); } else { throw new IOException("No FLINK_HOME is specified"); } } - private String getPyFlinkPythonPath(String pyFlinkFolder) { + private String getPyFlinkPythonPath(File pyFlinkFolder) throws IOException { LOGGER.info("Getting
[zeppelin] branch master updated: [ZEPPELIN-5621] add configuration of tmp_path to store compiled files like scala_shell_tmp-xxx.jar (#4328)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 80de2c8 [ZEPPELIN-5621] add configuration of tmp_path to store compiled files like scala_shell_tmp-xxx.jar (#4328) 80de2c8 is described below commit 80de2c811d1367daad975faa069f24d06871522c Author: Jeff Zhang AuthorDate: Thu Mar 31 15:47:23 2022 +0800 [ZEPPELIN-5621] add configuration of tmp_path to store compiled files like scala_shell_tmp-xxx.jar (#4328) * [ZEPPELIN-5621] add configuration of tmp_path to store compiled files like scala_shell_tmp-xxx.jar * Address comment * fix ci --- docs/interpreter/flink.md | 6 +- .../src/main/resources/interpreter-setting.json| 7 +++ .../apache/zeppelin/flink/internal/FlinkILoop.scala| 18 +- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/docs/interpreter/flink.md b/docs/interpreter/flink.md index 8e0476c..029394c 100644 --- a/docs/interpreter/flink.md +++ b/docs/interpreter/flink.md @@ -285,7 +285,11 @@ You can also add and set other Flink properties which are not listed in the tabl true Whether display Scala shell output in colorful format - + +zeppelin.flink.scala.shell.tmp_dir + +Temp folder for storing scala shell compiled jar + zeppelin.flink.enableHive false diff --git a/flink/flink-scala-parent/src/main/resources/interpreter-setting.json b/flink/flink-scala-parent/src/main/resources/interpreter-setting.json index dd67130..9da270a 100644 --- a/flink/flink-scala-parent/src/main/resources/interpreter-setting.json +++ b/flink/flink-scala-parent/src/main/resources/interpreter-setting.json @@ -138,6 +138,13 @@ "description": "Whether display scala shell output in colorful format", "type": "checkbox" }, + "zeppelin.flink.scala.shell.tmp_dir": { +"envName": "zeppelin.flink.scala.shell.tmp_dir", +"propertyName": "zeppelin.flink.scala.shell.tmp_dir", +"defaultValue": "", +"description": "Temp folder for storing scala shell compiled jar", +"type": "string" + }, "zeppelin.flink.enableHive": { "envName": null, "propertyName": null, diff --git a/flink/flink-scala-parent/src/main/scala/org/apache/zeppelin/flink/internal/FlinkILoop.scala b/flink/flink-scala-parent/src/main/scala/org/apache/zeppelin/flink/internal/FlinkILoop.scala index 1be64ab..eedcbe4 100644 --- a/flink/flink-scala-parent/src/main/scala/org/apache/zeppelin/flink/internal/FlinkILoop.scala +++ b/flink/flink-scala-parent/src/main/scala/org/apache/zeppelin/flink/internal/FlinkILoop.scala @@ -22,14 +22,16 @@ package org.apache.zeppelin.flink.internal import org.apache.flink.configuration.Configuration import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment import org.apache.flink.util.AbstractID -import java.io.{BufferedReader, File, FileOutputStream, IOException} +import java.io.{BufferedReader, File, FileOutputStream, IOException} import org.apache.flink.streaming.api.environment.{StreamExecutionEnvironment => JStreamExecutionEnvironment} import org.apache.flink.api.java.{ExecutionEnvironment => JExecutionEnvironment} import org.apache.flink.api.scala.ExecutionEnvironment import org.apache.flink.core.execution.PipelineExecutorServiceLoader import org.apache.zeppelin.flink.{ApplicationModeExecutionEnvironment, ApplicationModeStreamEnvironment, FlinkScalaInterpreter} import FlinkShell.ExecutionMode +import org.apache.commons.lang.StringUtils +import org.slf4j.{Logger, LoggerFactory} import scala.tools.nsc.interpreter._ @@ -45,6 +47,7 @@ class FlinkILoop( flinkScalaInterpreter: FlinkScalaInterpreter) extends ILoop(in0, out0) { + private lazy val LOGGER: Logger = LoggerFactory.getLogger(getClass) // remote environment private val (remoteBenv: ScalaShellEnvironment, @@ -101,11 +104,16 @@ class FlinkILoop( private val tmpDirBase: File = { // get unique temporary folder: val abstractID: String = new AbstractID().toString -val tmpDir: File = new File( - System.getProperty("java.io.tmpdir"), - "scala_shell_tmp-" + abstractID) +var scalaShellTmpParentFolder = flinkScalaInterpreter.properties.getProperty("zeppelin.flink.scala.shell.tmp_dir") +if (StringUtils.isBlank(scalaShellTmpParentFolder)) { + scalaShellTmpParentFolder = System.getProperty("java.io.tmpdir") +} +val tmpDir: File = new File(scalaShellTmpParentFolder, "scala_shell_tmp-&quo
[zeppelin] branch master updated: [ZEPPELIN-5530] Support scala-2.13 for spark interpreter
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new c4c580a [ZEPPELIN-5530] Support scala-2.13 for spark interpreter c4c580a is described below commit c4c580a37fde649553d336984a94bcb1b2821201 Author: Jeff Zhang AuthorDate: Wed Mar 30 09:00:25 2022 +0800 [ZEPPELIN-5530] Support scala-2.13 for spark interpreter ### What is this PR for? Spark 3.2 starts to support scala-2.13, this PR is to support scala-2.13 on the zeppelin side. A new module `scala-2.13` is introduced in this PR. `SparkILoop` is copied from the spark project with minor modification. ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5530 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4322 from zjffdu/ZEPPELIN-5530 and squashes the following commits: 8836ae87b4 [Jeff Zhang] update workflow 6470d3cda3 [Jeff Zhang] [ZEPPELIN-5530] Support scala 2.13 in spark interpreter --- .github/workflows/core.yml | 16 +- spark/interpreter/pom.xml | 110 - .../spark/AbstractSparkScalaInterpreter.java | 5 + .../zeppelin/spark/KotlinSparkInterpreter.java | 37 ++--- .../apache/zeppelin/spark/SparkInterpreter.java| 20 +++ .../apache/zeppelin/spark/SparkSqlInterpreter.java | 6 +- .../zeppelin/spark/SparkInterpreterTest.java | 132 --- spark/pom.xml | 77 + .../zeppelin/spark/SparkScala211Interpreter.scala | 73 - .../zeppelin/spark/SparkScala212Interpreter.scala | 70 +++- spark/scala-2.13/pom.xml | 59 +++ spark/scala-2.13/spark-scala-parent| 1 + .../scala-2.13/src/main/resources/log4j.properties | 50 ++ .../org/apache/zeppelin/spark/SparkILoop.scala | 87 ++ .../zeppelin/spark/SparkScala213Interpreter.scala | 179 + .../zeppelin/spark/BaseSparkScalaInterpreter.scala | 141 +++- zeppelin-interpreter-parent/pom.xml| 12 +- zeppelin-jupyter-interpreter-shaded/pom.xml| 8 + .../launcher/SparkInterpreterLauncher.java | 7 +- 19 files changed, 775 insertions(+), 315 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 578414e..d4d9b95 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -54,7 +54,7 @@ jobs: restore-keys: | ${{ runner.os }}-zeppelin- - name: install application with some interpreter -run: ./mvnw install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.11,spark/scala-2.12,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -B +run: ./mvnw install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.11,spark/scala-2.12,spark/scala-2.13,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -B - name: install and test plugins run: ./mvnw package -DskipRat -pl zeppelin-plugins -amd -B - name: Setup conda environment with python 3.7 and R @@ -74,7 +74,7 @@ jobs: conda list conda info - name: run tests with ${{ matrix.hadoop }} # skip spark test because we would run them in other CI -run: ./mvnw verify -Pusing-packaged-distr -DskipRat -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.11,spark/scala-2.12,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false +run: ./mvnw verify -Pusing-packaged-distr -DskipRat -pl zeppelin-server,zeppelin-web,spark-submit,spark/scala-2.11,spark/scala-2.12,spark/scala-2.13,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -Dtests.to.exclude=**/org/apache/zeppelin/spark/* -DfailIfNoTests=false # test interpreter modules except spark, flink, python, rlang, jupyter interpreter-test-non-core: @@ -190,7 +190,7 @@ jobs: ${{ runner.os }}-zeppelin- - name: install environment run: | - ./mvnw install -DskipTests -DskipRat -Phadoop2 -Pintegration -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/scala-2.11,spark/scala-2.12,markdown,flink-cmd,flink/flink-scala-2.11,flink/flink-scala-2.12,jdbc,shell -am
[zeppelin] branch master updated: [ZEPPELIN-5692] upgrade geode due to cves
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new a750ba7 [ZEPPELIN-5692] upgrade geode due to cves a750ba7 is described below commit a750ba7b59a4f6d7ad5a9b61f6a8e6703f4c95d2 Author: PJ Fanning AuthorDate: Thu Mar 24 20:34:54 2022 +0100 [ZEPPELIN-5692] upgrade geode due to cves ### What is this PR for? upgrade geode due to cves ### What type of PR is it? Bug Fix ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN/ZEPPELIN-5692 ### How should this be tested? * Strongly recommended: add automated unit tests for any new or changed behavior * Outline any manual steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: PJ Fanning Closes #4330 from pjfanning/ZEPPELIN-5692-geode and squashes the following commits: f7759cb954 [PJ Fanning] upgrade geode due to cves --- geode/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/geode/pom.xml b/geode/pom.xml index edd52a0..c521581 100644 --- a/geode/pom.xml +++ b/geode/pom.xml @@ -34,7 +34,7 @@ geode -1.1.0 +1.12.5
[zeppelin] branch master updated: [ZEPPELIN-5027] upgrade jackson due to CVEs
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new caeb9e2 [ZEPPELIN-5027] upgrade jackson due to CVEs caeb9e2 is described below commit caeb9e2e08d8f19c44abe7a9695d05eb690a7298 Author: PJ Fanning AuthorDate: Mon Mar 21 13:00:56 2022 +0100 [ZEPPELIN-5027] upgrade jackson due to CVEs ### What is this PR for? Existing jackson dependencies have CVEs ### What type of PR is it? Bug Fix ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5027 ### How should this be tested? * Strongly recommended: add automated unit tests for any new or changed behavior * Outline any manual steps to test the PR here. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? * Is there breaking changes for older versions? * Does this needs documentation? Author: PJ Fanning Closes #4325 from pjfanning/patch-2 and squashes the following commits: 5e3603c618 [PJ Fanning] upgrade jackson used with neo4j c780799ec7 [PJ Fanning] Update LICENSE af6ce8ba99 [PJ Fanning] [ZEPPELIN-5027] upgrade jackson due to CVEs --- neo4j/pom.xml | 2 +- zeppelin-distribution/src/bin_license/LICENSE | 2 +- zeppelin-server/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/neo4j/pom.xml b/neo4j/pom.xml index 389d52c..93f695d 100644 --- a/neo4j/pom.xml +++ b/neo4j/pom.xml @@ -33,7 +33,7 @@ 4.1.1 - 2.10.3 + 2.10.5.1 neo4j diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index 583b8c0..a1d7b4f 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -53,7 +53,7 @@ The following components are provided under Apache License. (Apache 2.0) Google Guava (com.google.guava:guava:15.0 - https://code.google.com/p/guava-libraries/) (Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-core:2.9.10 - https://github.com/FasterXML/jackson-core) (Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-annotations:2.9.10 - https://github.com/FasterXML/jackson-core) -(Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-databind:2.9.10.6 - https://github.com/FasterXML/jackson-core) +(Apache 2.0) Jackson (com.fasterxml.jackson.core:jackson-databind:2.9.10.8 - https://github.com/FasterXML/jackson-core) (Apache 2.0) Jackson Mapper ASL (org.codehaus.jackson:jackson-mapper-asl:1.9.13 - https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-mapper-asl/1.9.13) (Apache 2.0) javax.servlet (org.eclipse.jetty.orbit:javax.servlet:jar:3.1.0.v201112011016 - http://www.eclipse.org/jetty) (Apache 2.0) Jackson (org.codehaus.jackson:jackson-core-asl:1.9.13 - http://jackson.codehaus.org/) diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml index 2099047..db0ba1e 100644 --- a/zeppelin-server/pom.xml +++ b/zeppelin-server/pom.xml @@ -38,7 +38,7 @@ 2.1 1.11 4.1.0 -2.9.10.6 +2.9.10.8 9.13 2.0.0-M15
[zeppelin] branch master updated: [ZEPPELIN-5693] IPythonInterpreterTest.testIPythonPlotting fails due to jinjia2 version upgrade
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new ade718c [ZEPPELIN-5693] IPythonInterpreterTest.testIPythonPlotting fails due to jinjia2 version upgrade ade718c is described below commit ade718c778612e87a5d91c492169a54003ae3091 Author: Jeff Zhang AuthorDate: Fri Mar 25 11:52:32 2022 +0800 [ZEPPELIN-5693] IPythonInterpreterTest.testIPythonPlotting fails due to jinjia2 version upgrade ### What is this PR for? Specify the jinjia2 version explicitly so that we won't get the error described in the jira. ### What type of PR is it? [Bug Fix ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5693 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4331 from zjffdu/ZEPPELIN-5693 and squashes the following commits: bff6509c3b [Jeff Zhang] [ZEPPELIN-5693] IPythonInterpreterTest.testIPythonPlotting fails due to jinjia2 version upgrade --- testing/env_python_3.7_with_R.yml | 1 + testing/env_python_3.8_with_R.yml | 1 + testing/env_python_3.yml | 1 + testing/env_python_3_with_R.yml| 1 + testing/env_python_3_with_R_and_tensorflow.yml | 1 + testing/env_python_3_with_flink_112.yml| 1 + testing/env_python_3_with_flink_113.yml| 1 + testing/env_python_3_with_flink_114.yml| 1 + 8 files changed, 8 insertions(+) diff --git a/testing/env_python_3.7_with_R.yml b/testing/env_python_3.7_with_R.yml index 64d9768..10d46aa 100644 --- a/testing/env_python_3.7_with_R.yml +++ b/testing/env_python_3.7_with_R.yml @@ -22,6 +22,7 @@ dependencies: - altair - vega_datasets - plotly + - jinja2=3.0.3 - pip - r-base=3 - r-data.table diff --git a/testing/env_python_3.8_with_R.yml b/testing/env_python_3.8_with_R.yml index 64d9768..10d46aa 100644 --- a/testing/env_python_3.8_with_R.yml +++ b/testing/env_python_3.8_with_R.yml @@ -22,6 +22,7 @@ dependencies: - altair - vega_datasets - plotly + - jinja2=3.0.3 - pip - r-base=3 - r-data.table diff --git a/testing/env_python_3.yml b/testing/env_python_3.yml index 84a6c71..f9d548e 100644 --- a/testing/env_python_3.yml +++ b/testing/env_python_3.yml @@ -18,6 +18,7 @@ dependencies: - bokeh=1.3.4 - panel=0.6.0 - holoviews=1.12.3 + - jinja2=3.0.3 - pip - pip: - bkzep==0.6.1 diff --git a/testing/env_python_3_with_R.yml b/testing/env_python_3_with_R.yml index 512e632..016e708 100644 --- a/testing/env_python_3_with_R.yml +++ b/testing/env_python_3_with_R.yml @@ -23,6 +23,7 @@ dependencies: - altair - vega_datasets - plotly + - jinja2=3.0.3 - pip - r-base=3 - r-data.table diff --git a/testing/env_python_3_with_R_and_tensorflow.yml b/testing/env_python_3_with_R_and_tensorflow.yml index fc49ca3..498a00d 100644 --- a/testing/env_python_3_with_R_and_tensorflow.yml +++ b/testing/env_python_3_with_R_and_tensorflow.yml @@ -23,6 +23,7 @@ dependencies: - altair - vega_datasets - plotly + - jinja2=3.0.3 - pip - r-base=3 - r-data.table diff --git a/testing/env_python_3_with_flink_112.yml b/testing/env_python_3_with_flink_112.yml index 37fa6e0..7f0824f 100644 --- a/testing/env_python_3_with_flink_112.yml +++ b/testing/env_python_3_with_flink_112.yml @@ -22,6 +22,7 @@ dependencies: - altair - vega_datasets - plotly + - jinja2=3.0.3 - pip - pip: - apache-flink==1.12.2 diff --git a/testing/env_python_3_with_flink_113.yml b/testing/env_python_3_with_flink_113.yml index 59e9b2c..61ba025 100644 --- a/testing/env_python_3_with_flink_113.yml +++ b/testing/env_python_3_with_flink_113.yml @@ -22,6 +22,7 @@ dependencies: - altair - vega_datasets - plotly + - jinja2=3.0.3 - pip - pip: - apache-flink==1.13.1 diff --git a/testing/env_python_3_with_flink_114.yml b/testing/env_python_3_with_flink_114.yml index ad316be..5edc11d 100644 --- a/testing/env_python_3_with_flink_114.yml +++ b/testing/env_python_3_with_flink_114.yml @@ -22,6 +22,7 @@ dependencies: - altair - vega_datasets - plotly + - jinja2=3.0.3 - pip - pip: - apache-flink==1.14.0
[zeppelin] branch master updated: [ZEPPELIN-5624] Check if the path directory is compliant. (#4282)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 15cbe32 [ZEPPELIN-5624] Check if the path directory is compliant. (#4282) 15cbe32 is described below commit 15cbe3260090738d26283a28512e61075b2ac6f0 Author: iiusky AuthorDate: Fri Mar 25 18:36:59 2022 +0800 [ZEPPELIN-5624] Check if the path directory is compliant. (#4282) * [ZEPPELIN-5624] Check if the path directory is compliant. --- .../org/apache/zeppelin/interpreter/InterpreterSettingManager.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java index 6d30006..cfdae70 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java @@ -1035,12 +1035,11 @@ public class InterpreterSettingManager implements NoteEventListener, ClusterEven // Cluster event accepting nodes do not need to save files repeatedly saveToFile(); } + File localRepoPath = new File(conf.getInterpreterLocalRepoPath()); + FileUtils.deleteDirectory(new File(localRepoPath, id)); removed = true; } -File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + id); -FileUtils.deleteDirectory(localRepoDir); - return removed; }
[zeppelin] branch master updated: Revert "bouncycastle 1.68 (#4316)"
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 49b2e59 Revert "bouncycastle 1.68 (#4316)" 49b2e59 is described below commit 49b2e59625f54e29d787e94c8483980a90952014 Author: Jeff Zhang AuthorDate: Thu Mar 24 16:14:50 2022 +0800 Revert "bouncycastle 1.68 (#4316)" This reverts commit 75452443e2c05c67211adeb0d39061ee74442212. --- pom.xml | 2 +- zeppelin-distribution/src/bin_license/LICENSE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 0254165..b4857d3 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 3.2.2 1.4 1.7.0 -1.68 +1.60 3.6.3 4.1.14 1.6.0 diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index 6b7f84e..583b8c0 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -293,7 +293,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (The MIT License) headroom.js 0.9.3 (https://github.com/WickyNilliams/headroom.js) - https://github.com/WickyNilliams/headroom.js/blob/master/LICENSE (The MIT License) angular-viewport-watch 0.135 (https://github.com/wix/angular-viewport-watch) - https://github.com/wix/angular-viewport-watch/blob/master/LICENSE (The MIT License) ansi-up 2.0.2 (https://github.com/drudru/ansi_up) - https://github.com/drudru/ansi_up#license -(The MIT License) bcpkix-jdk15on 1.68 (org.bouncycastle:bcpkix-jdk15on:1.60 https://github.com/bcgit/bc-java) - https://github.com/bcgit/bc-java/blob/master/LICENSE.html +(The MIT License) bcpkix-jdk15on 1.60 (org.bouncycastle:bcpkix-jdk15on:1.60 https://github.com/bcgit/bc-java) - https://github.com/bcgit/bc-java/blob/master/LICENSE.html (The MIT License) influxdb-client-java 1.4.0 (com.influxdb:influxdb-client-java:1.4.0 https://github.com/influxdata/influxdb-client-java) - https://github.com/influxdata/influxdb-client-java/blob/master/LICENSE
[zeppelin] branch master updated: [ZEPPELIN-5683] further update commons-compress dependency (#4319)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 771c907 [ZEPPELIN-5683] further update commons-compress dependency (#4319) 771c907 is described below commit 771c9076082949d5bcd904a83407b5c2ba529b1e Author: PJ Fanning AuthorDate: Wed Mar 23 11:41:57 2022 +0100 [ZEPPELIN-5683] further update commons-compress dependency (#4319) --- zeppelin-plugins/launcher/docker/pom.xml | 2 +- zeppelin-zengine/pom.xml | 6 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/zeppelin-plugins/launcher/docker/pom.xml b/zeppelin-plugins/launcher/docker/pom.xml index b7bedbc..5adec5e 100644 --- a/zeppelin-plugins/launcher/docker/pom.xml +++ b/zeppelin-plugins/launcher/docker/pom.xml @@ -59,7 +59,7 @@ org.apache.commons commons-compress - 1.14 + ${commons.compress.version} org.powermock diff --git a/zeppelin-zengine/pom.xml b/zeppelin-zengine/pom.xml index 515e7ac..da50c7e 100644 --- a/zeppelin-zengine/pom.xml +++ b/zeppelin-zengine/pom.xml @@ -182,6 +182,10 @@ org.codehaus.plexus plexus-utils + + org.apache.commons + commons-compress + @@ -253,7 +257,7 @@ org.apache.commons commons-compress - 1.5 + ${commons.compress.version}
[zeppelin] branch master updated: bouncycastle 1.68 (#4316)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 7545244 bouncycastle 1.68 (#4316) 7545244 is described below commit 75452443e2c05c67211adeb0d39061ee74442212 Author: PJ Fanning AuthorDate: Mon Mar 21 09:21:29 2022 +0100 bouncycastle 1.68 (#4316) --- pom.xml | 2 +- zeppelin-distribution/src/bin_license/LICENSE | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b4857d3..0254165 100644 --- a/pom.xml +++ b/pom.xml @@ -140,7 +140,7 @@ 3.2.2 1.4 1.7.0 -1.60 +1.68 3.6.3 4.1.14 1.6.0 diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index 583b8c0..6b7f84e 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -293,7 +293,7 @@ The text of each license is also included at licenses/LICENSE-[project]-[version (The MIT License) headroom.js 0.9.3 (https://github.com/WickyNilliams/headroom.js) - https://github.com/WickyNilliams/headroom.js/blob/master/LICENSE (The MIT License) angular-viewport-watch 0.135 (https://github.com/wix/angular-viewport-watch) - https://github.com/wix/angular-viewport-watch/blob/master/LICENSE (The MIT License) ansi-up 2.0.2 (https://github.com/drudru/ansi_up) - https://github.com/drudru/ansi_up#license -(The MIT License) bcpkix-jdk15on 1.60 (org.bouncycastle:bcpkix-jdk15on:1.60 https://github.com/bcgit/bc-java) - https://github.com/bcgit/bc-java/blob/master/LICENSE.html +(The MIT License) bcpkix-jdk15on 1.68 (org.bouncycastle:bcpkix-jdk15on:1.60 https://github.com/bcgit/bc-java) - https://github.com/bcgit/bc-java/blob/master/LICENSE.html (The MIT License) influxdb-client-java 1.4.0 (com.influxdb:influxdb-client-java:1.4.0 https://github.com/influxdata/influxdb-client-java) - https://github.com/influxdata/influxdb-client-java/blob/master/LICENSE
[zeppelin] branch master updated: [ZEPPELIN-5644] Reduce redundancy during the interpreter execution process (#4294)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 7aad7d9 [ZEPPELIN-5644] Reduce redundancy during the interpreter execution process (#4294) 7aad7d9 is described below commit 7aad7d92d255cf19a56272b2d3ac40f47373a7ec Author: Leomax_Sun <282130...@qq.com> AuthorDate: Fri Mar 18 07:58:25 2022 +0800 [ZEPPELIN-5644] Reduce redundancy during the interpreter execution process (#4294) --- .../apache/zeppelin/spark/SparkInterpreter.java| 57 +- .../zeppelin/spark/SparkInterpreterTest.java | 5 +- .../zeppelin/spark/SparkScala211Interpreter.scala | 1 + .../zeppelin/spark/SparkScala212Interpreter.scala | 1 + .../zeppelin/spark/BaseSparkScalaInterpreter.scala | 3 +- .../remote/RemoteInterpreterServer.java| 19 +++- .../remote/RemoteInterpreterServerTest.java| 32 +++- 7 files changed, 91 insertions(+), 27 deletions(-) diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index fef0998..b13d67f 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -66,6 +66,7 @@ public class SparkInterpreter extends AbstractInterpreter { } private static AtomicInteger SESSION_NUM = new AtomicInteger(0); + private static Class innerInterpreterClazz; private AbstractSparkScalaInterpreter innerInterpreter; private Map innerInterpreterClassMap = new HashMap<>(); private SparkContext sc; @@ -151,38 +152,45 @@ public class SparkInterpreter extends AbstractInterpreter { */ private AbstractSparkScalaInterpreter loadSparkScalaInterpreter(SparkConf conf) throws Exception { scalaVersion = extractScalaVersion(conf); -ClassLoader scalaInterpreterClassLoader = Thread.currentThread().getContextClassLoader(); - -String zeppelinHome = System.getenv("ZEPPELIN_HOME"); -if (zeppelinHome != null) { - // ZEPPELIN_HOME is null in yarn-cluster mode, load it directly via current ClassLoader. - // otherwise, load from the specific folder ZEPPELIN_HOME/interpreter/spark/scala- - - File scalaJarFolder = new File(zeppelinHome + "/interpreter/spark/scala-" + scalaVersion); - List urls = new ArrayList<>(); - for (File file : scalaJarFolder.listFiles()) { -LOGGER.debug("Add file " + file.getAbsolutePath() + " to classpath of spark scala interpreter: " -+ scalaJarFolder); -urls.add(file.toURI().toURL()); +// Make sure the innerInterpreter Class is loaded only once into JVM +// Use double lock to ensure thread safety +if (innerInterpreterClazz == null) { + synchronized (SparkInterpreter.class) { +if (innerInterpreterClazz == null) { + LOGGER.debug("innerInterpreterClazz is null, thread:{}", Thread.currentThread().getName()); + ClassLoader scalaInterpreterClassLoader = Thread.currentThread().getContextClassLoader(); + String zeppelinHome = System.getenv("ZEPPELIN_HOME"); + if (zeppelinHome != null) { +// ZEPPELIN_HOME is null in yarn-cluster mode, load it directly via current ClassLoader. +// otherwise, load from the specific folder ZEPPELIN_HOME/interpreter/spark/scala- +File scalaJarFolder = new File(zeppelinHome + "/interpreter/spark/scala-" + scalaVersion); +List urls = new ArrayList<>(); +for (File file : scalaJarFolder.listFiles()) { + LOGGER.debug("Add file " + file.getAbsolutePath() + " to classpath of spark scala interpreter: " + + scalaJarFolder); + urls.add(file.toURI().toURL()); +} +scalaInterpreterClassLoader = new URLClassLoader(urls.toArray(new URL[0]), +Thread.currentThread().getContextClassLoader()); + } + String innerIntpClassName = innerInterpreterClassMap.get(scalaVersion); + innerInterpreterClazz = scalaInterpreterClassLoader.loadClass(innerIntpClassName); +} } - scalaInterpreterClassLoader = new URLClassLoader(urls.toArray(new URL[0]), - Thread.currentThread().getContextClassLoader()); } - -String innerIntpClassName = innerInterpreterClassMap.get(scalaVersion); -Class clazz = scalaInterpreterClassLoader.loadClass(innerIntpClassName); return (AbstractSparkScalaInterpreter) -clazz.getConstructor(SparkConf.class, List.class, Properties.class, InterpreterGroup.class, URLClassLoader.clas
[zeppelin] branch master updated (812bc0f -> 9cecd23)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git. from 812bc0f [ZEPPELIN-5620] Staging directory is not deleted in Flink's yarn-application (#4306) add 9cecd23 [ZEPPELIN-5678] Remove the wordcount init job in Flink yarn-application mode (#4313) No new revisions were added by this update. Summary of changes: .../flink/ApplicationModeExecutionEnvironment.java| 7 +++ .../flink/ApplicationModeStreamEnvironment.java | 13 + .../apache/zeppelin/flink/FlinkScalaInterpreter.scala | 19 --- 3 files changed, 20 insertions(+), 19 deletions(-)
[zeppelin] branch master updated (be13f03 -> 812bc0f)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git. from be13f03 upgrade commons-compress and commons-io due to security concerns (#4315) add 812bc0f [ZEPPELIN-5620] Staging directory is not deleted in Flink's yarn-application (#4306) No new revisions were added by this update. Summary of changes: .../org/apache/zeppelin/flink/HadoopUtils.java | 5 ++--- .../zeppelin/flink/FlinkScalaInterpreter.scala | 23 +- .../zeppelin/flink/internal/FlinkShell.scala | 12 +-- 3 files changed, 26 insertions(+), 14 deletions(-)
[zeppelin] branch master updated: upgrade commons-compress and commons-io due to security concerns (#4315)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new be13f03 upgrade commons-compress and commons-io due to security concerns (#4315) be13f03 is described below commit be13f03f537a314657246d8aff719aed6691acd0 Author: PJ Fanning AuthorDate: Thu Mar 17 09:53:42 2022 +0100 upgrade commons-compress and commons-io due to security concerns (#4315) --- pom.xml | 4 ++-- zeppelin-distribution/src/bin_license/LICENSE | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pom.xml b/pom.xml index a8dee84..33541f4 100644 --- a/pom.xml +++ b/pom.xml @@ -130,13 +130,13 @@ 4.4.1 4.5.13 4.0.2 -1.20 +1.21 3.10 1.8 2.7 1.3 1.14 -2.6 +2.7 3.2.2 1.4 1.7.0 diff --git a/zeppelin-distribution/src/bin_license/LICENSE b/zeppelin-distribution/src/bin_license/LICENSE index c956727..583b8c0 100644 --- a/zeppelin-distribution/src/bin_license/LICENSE +++ b/zeppelin-distribution/src/bin_license/LICENSE @@ -6,8 +6,8 @@ The following components are provided under Apache License. (Apache 2.0) JavaEWAH v0.7.9 (https://github.com/lemire/javaewah) - https://github.com/lemire/javaewah/blob/master/LICENSE-2.0.txt (Apache 2.0) Apache Commons Logging (commons-logging:commons-logging:1.1.1 - http://commons.apache.org/proper/commons-logging/) (Apache 2.0) Apache Commons Codec (commons-codec:commons-codec:1.5 - http://commons.apache.org/proper/commons-codec/) -(Apache 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.1 - http://commons.apache.org/proper/commons-configuration/) -(Apache 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.9 - http://commons.apache.org/proper/commons-compress/) +(Apache 2.0) Apache Commons Collections (commons-collections:commons-collections:3.2.2 - http://commons.apache.org/proper/commons-configuration/) +(Apache 2.0) Apache Commons Compress (org.apache.commons:commons-compress:1.21 - http://commons.apache.org/proper/commons-compress/) (Apache 2.0) Apache Commons Configuration (org.apache.commons:commons-configuration2:2.7 - http://commons.apache.org/configuration/) (Apache 2.0) Apache Commons CLI (commons-cli:commons-cli:1.2 - http://commons.apache.org/cli/) (Apache 2.0) Apache Commons Exec (commons-exec:commons-exec:1.3 - http://commons.apache.org/exec/) @@ -20,7 +20,7 @@ The following components are provided under Apache License. (Apache 2.0) Apache Commons Net (commons-net:commons-net:2.2 - http://commons.apache.org/proper/commons-net/) (Apache 2.0) Apache Commons Pool2 (commons-exec:commons-pool2:2.3 - https://commons.apache.org/proper/commons-pool/) (Apache 2.0) Apache Commons FileUpload (commons-fileupload:commons-fileupload:1.3.1 - http://commons.apache.org/fileupload/) -(Apache 2.0) Apache Commons IO (commons-io:commons-io:2.4 - http://commons.apache.org/io/) +(Apache 2.0) Apache Commons IO (commons-io:commons-io:2.7 - http://commons.apache.org/io/) (Apache 2.0) Apache Commons VFS2 (org.apache.commons:commons-vfs2:2.0 - https://commons.apache.org/proper/commons-vfs/) (Apache 2.0) Apache Jackrabbit webdav (org.apache.jackrabbit:jackrabbit-webdav:jar:1.5.2 - https://jackrabbit.apache.org/jcr/components/jackrabbit-webdav-library.html) (Apache 2.0) Apache Jackrabbit JCR commons - http://jackrabbit.apache.org/jcr/components/jackrabbit-jcr-commons.html)
[zeppelin] branch master updated: [ZEPPELIN-5671] Split spark-integration-test to hadoop2 and hadoop3 (#4311)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new d6da9af [ZEPPELIN-5671] Split spark-integration-test to hadoop2 and hadoop3 (#4311) d6da9af is described below commit d6da9af4abbaeb71d93b5273df423af808a035f1 Author: Jeff Zhang AuthorDate: Wed Mar 16 22:15:31 2022 +0800 [ZEPPELIN-5671] Split spark-integration-test to hadoop2 and hadoop3 (#4311) --- .github/workflows/core.yml | 12 ++-- 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index e765a54..f9eb3ce 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -254,6 +254,10 @@ jobs: spark-integration-test: runs-on: ubuntu-20.04 +strategy: + fail-fast: false + matrix: +hadoop: [ 2, 3 ] steps: - name: Checkout uses: actions/checkout@v2 @@ -291,12 +295,8 @@ jobs: - name: Make IRkernel available to Jupyter run: | R -e "IRkernel::installspec()" - - name: run tests on hadoop2 -run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration -Phadoop2 -Pintegration -B -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest30,SparkIntegrationTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest31,ZeppelinSparkClusterTest32,SparkIntegrationTest32 -DfailIfNoTests=false - - name: run tests on hadoop3 -run: | - rm -rf spark/interpreter/metastore_db - ./mvnw test -DskipRat -pl zeppelin-interpreter-integration -Phadoop3 -Pintegration -B -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest30,SparkIntegrationTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest31,ZeppelinSparkClusterTest32,SparkIntegrationTest32 -DfailIfNoTests=false + - name: run tests on hadoop${{ matrix.hadoop }} +run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration -Phadoop${{ matrix.hadoop }} -Pintegration -B -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest30,SparkIntegrationTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest31,ZeppelinSparkClusterTest32,SparkIntegrationTest32 -DfailIfNoTests=false # test on spark for each spark version & scala version spark-test:
[zeppelin] branch master updated (bea51d1 -> ba06bae)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git. from bea51d1 rework xml factory (#4302) add ba06bae [ZEPPELIN-5670] Frontend CI fails (#4310) No new revisions were added by this update. Summary of changes: .github/workflows/frontend.yml | 8 +- zeppelin-web/e2e/collaborativeMode.spec.js | 143 +++-- .../launcher/SparkInterpreterLauncher.java | 10 +- 3 files changed, 86 insertions(+), 75 deletions(-)
[zeppelin] branch master updated (c26ed1a -> bea51d1)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git. from c26ed1a [ZEPPELIN-5661] Support clone a specific revision of a note (#4299) add bea51d1 rework xml factory (#4302) No new revisions were added by this update. Summary of changes: .../zeppelin/sap/universe/UniverseClient.java | 44 ++ 1 file changed, 36 insertions(+), 8 deletions(-)
[zeppelin] branch master updated: [ZEPPELIN-5661] Support clone a specific revision of a note (#4299)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new c26ed1a [ZEPPELIN-5661] Support clone a specific revision of a note (#4299) c26ed1a is described below commit c26ed1a0bc0221141d8d216c54507a72a6580e2a Author: nihua AuthorDate: Mon Mar 14 16:11:09 2022 +0800 [ZEPPELIN-5661] Support clone a specific revision of a note (#4299) --- docs/usage/rest_api/notebook.md| 8 ++- .../org/apache/zeppelin/rest/NotebookRestApi.java | 4 +- .../zeppelin/rest/message/NewNoteRequest.java | 5 ++ .../apache/zeppelin/service/NotebookService.java | 23 +-- .../apache/zeppelin/rest/NotebookRestApiTest.java | 79 +++--- .../org/apache/zeppelin/notebook/Notebook.java | 32 ++--- 6 files changed, 113 insertions(+), 38 deletions(-) diff --git a/docs/usage/rest_api/notebook.md b/docs/usage/rest_api/notebook.md index c49d805..b669213 100644 --- a/docs/usage/rest_api/notebook.md +++ b/docs/usage/rest_api/notebook.md @@ -352,7 +352,8 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete, Description This ```POST``` method clones a note by the given id and create a new note using the given name - or default name if none given. + or default name if none given. If what you want to copy is a certain version of note, you need + to specify the revisionId. The body field of the returned JSON contains the new note id. @@ -373,7 +374,10 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete, ```json -{"name": "name of new note"} +{ + "name": "name of new note", + "revisionId": "revisionId of note to be copied (optional)" +} ``` diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index 40ad46a..88004f4 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -555,11 +555,13 @@ public class NotebookRestApi extends AbstractRestApi { checkIfUserCanWrite(noteId, "Insufficient privileges you cannot clone this note"); NewNoteRequest request = NewNoteRequest.fromJson(message); String newNoteName = null; +String revisionId = null; if (request != null) { newNoteName = request.getName(); + revisionId = request.getRevisionId(); } AuthenticationInfo subject = new AuthenticationInfo(authenticationService.getPrincipal()); -String newNoteId = notebookService.cloneNote(noteId, newNoteName, getServiceContext(), +String newNoteId = notebookService.cloneNote(noteId, revisionId, newNoteName, getServiceContext(), new RestServiceCallback() { @Override public void onSuccess(Note newNote, ServiceContext context) throws IOException { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java index 85744dc..1ef9da3 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewNoteRequest.java @@ -33,6 +33,7 @@ public class NewNoteRequest implements JsonSerializable { private String defaultInterpreterGroup; private boolean addingEmptyParagraph = false; private List paragraphs; + private String revisionId; public NewNoteRequest (){ } @@ -53,6 +54,10 @@ public class NewNoteRequest implements JsonSerializable { return paragraphs; } + public String getRevisionId() { +return revisionId; + } + public String toJson() { return GSON.toJson(this); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java index 02a4bbd..dfc75eb 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java @@ -317,15 +317,30 @@ public class NotebookService { } public String cloneNote(String noteId, -String newNotePath, -ServiceContext context, -ServiceCallback callback) throws IOException { + String newNotePath, + ServiceContext context, + ServiceCallback callback) throws IOException { +return cloneNote(noteId, &q
[zeppelin] branch master updated: [ZEPPELIN-5663] Provide REST API about notebook version control (#4300)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 60ddde6 [ZEPPELIN-5663] Provide REST API about notebook version control (#4300) 60ddde6 is described below commit 60ddde694681f2071e60b21dfd600c4831b0aad4 Author: nihua AuthorDate: Mon Mar 14 16:09:56 2022 +0800 [ZEPPELIN-5663] Provide REST API about notebook version control (#4300) * [ZEPPELIN-5663] Provide REST API about notebook version control * change RESTFUL API method checkpointNote(...) to return revisionId like that addParagraph return paragraphId --- docs/usage/rest_api/notebook.md| 238 + .../org/apache/zeppelin/rest/NotebookRestApi.java | 88 +++- .../rest/message/CheckpointNoteRequest.java| 28 +++ .../apache/zeppelin/service/NotebookService.java | 10 +- .../apache/zeppelin/rest/NotebookRestApiTest.java | 200 + 5 files changed, 559 insertions(+), 5 deletions(-) diff --git a/docs/usage/rest_api/notebook.md b/docs/usage/rest_api/notebook.md index b0dcd46..c49d805 100644 --- a/docs/usage/rest_api/notebook.md +++ b/docs/usage/rest_api/notebook.md @@ -1574,3 +1574,241 @@ Notebooks REST API supports the following operations: List, Create, Get, Delete, +## Version control + + + +### Get revisions of a note + + + + + Description + This ```GET``` method gets the revisions of a note. + + + + URL + ```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[noteId]/revision``` + + + Success code + 200 + + + Fail code + 500 + + + sample JSON response + + +```json +{ +"status": "OK", +"body": [ +{ +"id": "f97ce5c7f076783023d33623ad52ca994277e5c1", +"message": "first commit", +"time": 1645712061 +}, +{ +"id": "e9b964bebdecec6a59efe085f97db4040ae333aa", +"message": "second commit", +"time": 1645693163 +} +] +} +``` + + + + + +### Save a revision for a note + + + + Description + This ```POST``` method saves a revision for a note. + + + + URL + ```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[noteId]/revision``` + + + Success code + 200 + + + Bad Request code + 400 + + + Fail code + 500 + + + sample JSON input + + +```json +{ + "commitMessage": "first commit" +} +``` + + + + sample JSON response + + +```json +{ + "status": "OK", + "message": "", + "body": "6a5879218dfb797b013bcd24a594808045e34875" +} +``` + + + +### Get a revision of a note + + + + Description + This ```GET``` method gets a revision of a note. + + + + URL + ```http://[zeppelin-server]:[zeppelin-port]/api/notebook/[noteId]/revision/{revisionId}``` + + + Success code + 200 + + + Fail code + 500 + + + sample JSON response + + +```json +{ + "status": "OK", + "message": "", + "body": { +"paragraphs": [ + { +"text": "%sql \nselect age, count(1) value\nfrom bank \nwhere age < 30 \ngroup by + age \norder by age", +"config": { + "colWidth": 4, + "graph": { +"mode": "multiBarChart", +"height": 300, +"optionOpen": false, +"keys": [ + { +"name": "age", +"index": 0, +"aggr": "sum" + } +], +"values": [ + { +"name": "value", +"index": 1, +"aggr": "sum" + } +], +"groups": [], +"scatter": { + "xAxis": { +"name": "age", +"index": 0, +"aggr": "sum" + }, + "yAxis": { +"name": "value", +"index": 1, +"aggr": "sum" + } +} + } +}, +
[zeppelin] branch master updated: [ZEPPELIN-5433] Update httpClient version (#4303)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 78b7b3a [ZEPPELIN-5433] Update httpClient version (#4303) 78b7b3a is described below commit 78b7b3ad3377deabfd355223cdd21d2470b4ca54 Author: nihua AuthorDate: Wed Mar 9 15:59:30 2022 +0800 [ZEPPELIN-5433] Update httpClient version (#4303) --- jdbc/pom.xml | 2 +- pom.xml | 2 +- rlang/pom.xml | 2 +- sap/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jdbc/pom.xml b/jdbc/pom.xml index 86c1f85..6afdc34 100644 --- a/jdbc/pom.xml +++ b/jdbc/pom.xml @@ -178,7 +178,7 @@ org.apache.httpcomponents httpclient - 4.4.1 + 4.5.13 diff --git a/pom.xml b/pom.xml index 8d0754c..a8dee84 100644 --- a/pom.xml +++ b/pom.xml @@ -128,7 +128,7 @@ 0.2.2 9.4.31.v20200723 4.4.1 -4.5.1 +4.5.13 4.0.2 1.20 3.10 diff --git a/rlang/pom.xml b/rlang/pom.xml index 0037d7d..0fec924 100644 --- a/rlang/pom.xml +++ b/rlang/pom.xml @@ -91,7 +91,7 @@ org.apache.httpcomponents httpclient -4.5.1 +4.5.13 diff --git a/sap/pom.xml b/sap/pom.xml index 6f6074b..40d1d9b 100644 --- a/sap/pom.xml +++ b/sap/pom.xml @@ -46,7 +46,7 @@ org.apache.httpcomponents httpclient -4.5.1 +4.5.13
svn commit: r1898655 - in /zeppelin/site/docs/0.10.1: ./ development/ development/contribution/ development/helium/ interpreter/ quickstart/ setup/basics/ setup/deployment/ setup/operation/ setup/secu
Author: zjffdu Date: Sun Mar 6 14:18:28 2022 New Revision: 1898655 URL: http://svn.apache.org/viewvc?rev=1898655=rev Log: Update 0.10.1 doc [This commit notification would consist of 52 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.]
[zeppelin] branch master updated: [ZEPPELIN-5660] Implement move gcsnotebookrepo folder (#4298)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 2b4b657 [ZEPPELIN-5660] Implement move gcsnotebookrepo folder (#4298) 2b4b657 is described below commit 2b4b6570e1ceb921792b3a13973288a18076903d Author: Bagus Nugroho Budi Nurtomo AuthorDate: Wed Mar 2 15:17:56 2022 +0700 [ZEPPELIN-5660] Implement move gcsnotebookrepo folder (#4298) * Implement Move and Remove folders for GCSNotebookRepo * Fix tests using wrong parameters * Small refactor * Updated tests --- .../zeppelin/notebook/repo/GCSNotebookRepo.java| 58 -- .../notebook/repo/GCSNotebookRepoTest.java | 43 +++- 2 files changed, 97 insertions(+), 4 deletions(-) diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java index 0f2aed3..083f564 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java @@ -33,6 +33,7 @@ import com.google.gson.JsonParseException; import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -214,8 +215,37 @@ public class GCSNotebookRepo implements NotebookRepo { } @Override - public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) { + public void move(String folderPath, String newFolderPath, AuthenticationInfo subject) throws IOException{ +if(!folderPath.endsWith("/")) { + folderPath = folderPath + "/"; +} +if(!newFolderPath.endsWith("/")) { + newFolderPath = newFolderPath + "/"; +} +if(basePath.isPresent()) { + folderPath = basePath.get() + folderPath; + newFolderPath = basePath.get() + newFolderPath; +} +String oldPath = folderPath; +String newPath = newFolderPath; +try { + ArrayList toBeDeleted = new ArrayList(); + storage.list(bucketName, Storage.BlobListOption.prefix(oldPath)).getValues() + .forEach((note -> { +toBeDeleted.add(note.getBlobId()); + })); + if(toBeDeleted.isEmpty()) { +throw new IOException("Empty folder or folder does not exist: " + oldPath); + } + for(BlobId note: toBeDeleted) { +String newLocation = ("/" + note.getName()).replaceFirst(oldPath, newPath).substring(1); +storage.get(note).copyTo(bucketName, newLocation); +storage.delete(note); + }; +} catch (Exception se) { + throw new IOException("Could not copy from " + oldPath + " to " + newPath + ": " + se.getMessage(), se); +} } @Override @@ -233,8 +263,30 @@ public class GCSNotebookRepo implements NotebookRepo { } @Override - public void remove(String folderPath, AuthenticationInfo subject) { - + public void remove(String folderPath, AuthenticationInfo subject) throws IOException { +if(!folderPath.endsWith("/")) { + folderPath = folderPath + "/"; +} +if(basePath.isPresent()) { + folderPath = basePath.get() + folderPath; +} +String oldPath = folderPath; +try { + ArrayList toBeDeleted = new ArrayList(); + storage.list(bucketName, Storage.BlobListOption.prefix(oldPath)).getValues() + .forEach((note -> { +toBeDeleted.add(note.getBlobId()); + })); + if(toBeDeleted.isEmpty()) { +throw new IOException("Empty folder or folder does not exist: " + oldPath); + } + // Note(Bagus): We an actually do this with storage.delete(toBeDeleted) but FakeStorageRPC used for tests still does not support it + for(BlobId note: toBeDeleted) { +storage.delete(note); + } +} catch (Exception se) { + throw new IOException("Could not delete from " + oldPath + ": " + se.getMessage(), se); +} } @Override diff --git a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java index 69ef76f..86c6034 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo
[zeppelin] branch gh-pages updated (a4a3090 -> 0559ee1)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to branch gh-pages in repository https://gitbox.apache.org/repos/asf/zeppelin.git. from a4a3090 update doc new f98f44a update doc for 0.10 new 0559ee1 Update doc for 0.10.1 The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .htaccess | 4 +- _includes/themes/zeppelin/_navigation.html | 15 +- doap.rdf | 14 ++ documentation.md | 2 + download.md| 329 ++--- index.md | 2 +- whats_new.md | 71 --- 7 files changed, 131 insertions(+), 306 deletions(-)
[zeppelin] 01/02: update doc for 0.10
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch gh-pages in repository https://gitbox.apache.org/repos/asf/zeppelin.git commit f98f44abc8d1b2640560998fcaac9e9ce86aca2c Author: jeffzhang.zjf AuthorDate: Fri Aug 20 14:32:54 2021 +0800 update doc for 0.10 --- _includes/themes/zeppelin/_navigation.html | 10 +- documentation.md | 1 + download.md| 267 +++-- whats_new.md | 71 +--- 4 files changed, 68 insertions(+), 281 deletions(-) diff --git a/_includes/themes/zeppelin/_navigation.html b/_includes/themes/zeppelin/_navigation.html index 90591ec..92167f4 100644 --- a/_includes/themes/zeppelin/_navigation.html +++ b/_includes/themes/zeppelin/_navigation.html @@ -14,7 +14,7 @@ -Quick Start +Quick Start Download @@ -30,16 +30,10 @@ Release 0.9.0 0.8.2 -0.8.1 -0.8.0 -0.7.3 -0.7.2 -0.6.2 -0.6.1 Older Versions Snapshot(development) -0.9.0-SNAPSHOT +0.10.0-SNAPSHOT diff --git a/documentation.md b/documentation.md index e50344a..2e6efcb 100644 --- a/documentation.md +++ b/documentation.md @@ -20,6 +20,7 @@ limitations under the License. # Apache Zeppelin Documentation + * [Zeppelin 0.10.0](docs/0.10.0) * [Zeppelin 0.9.0](docs/0.9.0) * [Zeppelin 0.8.2](docs/0.8.2) * [Zeppelin 0.8.1](docs/0.8.1) diff --git a/download.md b/download.md index caa15ea..5b4767d 100644 --- a/download.md +++ b/download.md @@ -74,6 +74,26 @@ For developers, to get latest *0.9.0-SNAPSHOT* check [README](https://github.com ## Old releases + + - 0.9.0 released on Dec 26, 2020 ([release notes](./releases/zeppelin-release-0.9.0.html)) ([git tag](https://gitbox.apache.org/repos/asf?p=zeppelin.git;h=refs/tags/v0.9.0)) + +* Binary package with all interpreters ([Install guide](../../docs/0.9.0/quickstart/install.html)): + http://www.apache.org/dyn/closer.cgi/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-all.tgz'">zeppelin-0.9.0-bin-all.tgz (1.5g, + [pgp](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-all.tgz.asc), + [md5](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-all.tgz.md5), + [sha](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-all.tgz.sha512)) + +* Binary package with Spark interpreter and interpreter net-install script ([interpreter installation guide](../../docs/0.9.0/usage/interpreter/installation.html)): + http://www.apache.org/dyn/closer.cgi/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-netinst.tgz'">zeppelin-0.9.0-bin-netinst.tgz (568 MB, + [pgp](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-netinst.tgz.asc), + [md5](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-netinst.tgz.md5), + [sha](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-netinst.tgz.sha512)) + +* Source: + http://www.apache.org/dyn/closer.cgi/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0.tgz'">zeppelin-0.9.0.tgz (9.1 MB, + [pgp](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0.tgz.asc), + [md5](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0.tgz.md5), + [sha](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0.tgz.sha512)) - 0.8.2 released on Sep 29, 2018 ([release notes](./releases/zeppelin-release-0.8.2.html)) ([git tag](https://git-wip-us.apache.org/repos/asf?p=zeppelin.git;h=refs/tags/v0.8.2)) @@ -94,253 +114,6 @@ For developers, to get latest *0.9.0-SNAPSHOT* check [README](https://github.com [pgp](https://www.apache.org/dist/zeppelin/zeppelin-0.8.2/zeppelin-0.8.2.tgz.asc), [md5](https://www.apache.org/dist/zeppelin/zeppelin-0.8.2/zeppelin-0.8.2.tgz.md5), [sha](https://www.apache.org/dist/zeppelin/zeppelin-0.8.2/zeppelin-0.8.2.tgz.sha512)) - - - 0.8.1 released on Jan 29, 2018 ([release notes](./releases/zeppelin-release-0.8.1.html)) ([git tag](https://git-wip-us.apache.org/repos/asf?p=zeppelin.git;h=refs/tags/v0.8.0)) - -* Binary package with all interpreters ([Install guide](../../docs/0.8.1/quickstart/install.html)): -http://www.apache.org/dyn/closer.cgi/zeppelin/zeppelin-0.8.1/zeppelin-0.8.1-bin-all.tgz'">zeppelin-0.8.1-bin-all.tgz (939 MB, - [pgp](https://www.apache.org/dist/zeppelin/zeppelin-0.8.1/zeppelin-0.8.1-bin-all.tgz.asc), - [md5](https://www.apache.org/dist/zeppelin/zeppelin-0.8.1/zeppelin-0.8.1-bin-all.tgz.md5), - [sha](https://www.apache.org/dist/zeppelin/zeppelin-0.8.1/zeppelin-0.8.1-bin-all.tgz.sha512)) - -* Binary package with Spark interpreter an
[zeppelin] 02/02: Update doc for 0.10.1
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch gh-pages in repository https://gitbox.apache.org/repos/asf/zeppelin.git commit 0559ee1b91adebd1b04d825b642066d520976eb9 Author: Jeff Zhang AuthorDate: Tue Mar 1 16:20:44 2022 +0800 Update doc for 0.10.1 --- .htaccess | 4 +- _includes/themes/zeppelin/_navigation.html | 7 ++-- doap.rdf | 14 +++ documentation.md | 1 + download.md| 62 +- index.md | 2 +- whats_new.md | 14 +++ 7 files changed, 71 insertions(+), 33 deletions(-) diff --git a/.htaccess b/.htaccess index 681de07..cfd15b2 100644 --- a/.htaccess +++ b/.htaccess @@ -7,7 +7,7 @@ RewriteRule ^/?docs/0.6.0-(?!SNAPSHOT).*/(.*) /docs/0.6.0/$2 [R=301,L,NE] RewriteRule ^/?docs/(0.5.[056])(?!-incubating).*/(.*) /docs/$1-incubating/$3 [R=301,L,NE] # rewrite docs/latest to latest stable release -RewriteRule ^/?docs/latest/(.*) /docs/0.9.0/$1 [PT] +RewriteRule ^/?docs/latest/(.*) /docs/0.10.1/$1 [PT] # rewrite docs/snapshot to latest snapshot version -RewriteRule ^/?docs/snapshot/(.*) /docs/0.9.0-SNAPSHOT/$1 [PT] +RewriteRule ^/?docs/snapshot/(.*) /docs/0.10.1-SNAPSHOT/$1 [PT] diff --git a/_includes/themes/zeppelin/_navigation.html b/_includes/themes/zeppelin/_navigation.html index 92167f4..a77c5d6 100644 --- a/_includes/themes/zeppelin/_navigation.html +++ b/_includes/themes/zeppelin/_navigation.html @@ -28,12 +28,11 @@ Docs Release +0.10.1 +0.10.0 0.9.0 0.8.2 Older Versions - - Snapshot(development) -0.10.0-SNAPSHOT @@ -74,7 +73,7 @@ Apache Zeppelin Web-based notebook that enables data-driven, - interactive data analytics and collaborative documents with SQL, Scala and more. + interactive data analytics and collaborative documents with SQL, Scala, Python, R and more. diff --git a/doap.rdf b/doap.rdf index 21bfbb8..c7005dd 100644 --- a/doap.rdf +++ b/doap.rdf @@ -38,6 +38,20 @@ http://projects.apache.org/category/big-data; /> +0.10.1 +2022-02-28 +0.10.1 + + + + +0.10.0 +2021-08-24 +0.10.0 + + + + 0.9.0 2020-12-24 0.9.0 diff --git a/documentation.md b/documentation.md index 2e6efcb..4d33f01 100644 --- a/documentation.md +++ b/documentation.md @@ -20,6 +20,7 @@ limitations under the License. # Apache Zeppelin Documentation + * [Zeppelin 0.10.1](docs/0.10.1) * [Zeppelin 0.10.0](docs/0.10.0) * [Zeppelin 0.9.0](docs/0.9.0) * [Zeppelin 0.8.2](docs/0.8.2) diff --git a/download.md b/download.md index 5b4767d..179dcf6 100644 --- a/download.md +++ b/download.md @@ -21,27 +21,27 @@ limitations under the License. # Download Apache Zeppelin -The latest release of Apache Zeppelin is **0.9.0**. +The latest release of Apache Zeppelin is **0.10.1**. - - 0.9.0 released on Dec 26, 2020 ([release notes](./releases/zeppelin-release-0.9.0.html)) ([git tag](https://gitbox.apache.org/repos/asf?p=zeppelin.git;h=refs/tags/v0.9.0)) + - 0.10.1 released on Feb 29, 2022 ([release notes](./releases/zeppelin-release-0.10.1.html)) ([git tag](https://gitbox.apache.org/repos/asf?p=zeppelin.git;h=refs/tags/v0.11.0)) -* Binary package with all interpreters ([Install guide](../../docs/0.9.0/quickstart/install.html)): -http://www.apache.org/dyn/closer.cgi/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-all.tgz'">zeppelin-0.9.0-bin-all.tgz (1.5g, - [pgp](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-all.tgz.asc), - [md5](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-all.tgz.md5), - [sha](https://www.apache.org/dist/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-all.tgz.sha512)) +* Binary package with all interpreters ([Install guide](../../docs/0.10.1/quickstart/install.html)): +http://www.apache.org/dyn/closer.cgi/zeppelin/zeppelin-0.10.1/zeppelin-0.10.1-bin-all.tgz'">zeppelin-0.10.1-bin-all.tgz (1.5g, + [pgp](https://www.apache.org/dist/zeppelin/zeppelin-0.10.1/zeppelin-0.10.1-bin-all.tgz.asc), + [md5](https://www.apache.org/dist/zeppelin/zeppelin-0.10.1/zeppelin-0.10.1-bin-all.tgz.md5), + [sha](https://www.apache.org/dist/zeppelin/zeppelin-0.10.1/zeppelin-0.10.1-bin-all.tgz.sha512)) -* Binary package with Spark interpreter and interpreter net-install script ([interpreter installation guide](../../docs/0.9.0/usage/interpreter/installation.html)): -http://www.apache.org/dyn/closer.cgi/zeppelin/zeppelin-0.9.0/zeppelin-0.9.0-bin-netinst.tgz'&
svn commit: r1898499 - in /zeppelin/site/docs/0.10.1: ./ assets/ assets/themes/ assets/themes/zeppelin/ assets/themes/zeppelin/bootstrap/ assets/themes/zeppelin/bootstrap/css/ assets/themes/zeppelin/b
Author: zjffdu Date: Tue Mar 1 08:18:42 2022 New Revision: 1898499 URL: http://svn.apache.org/viewvc?rev=1898499=rev Log: Publish document 0.10.1 [This commit notification would consist of 52 parts, which exceeds the limit of 50 ones, so it was shortened to the summary.]
svn commit: r1898497 - in /zeppelin/site: ./ contribution/ contribution/zeppelinweb/ releases/
Author: zjffdu Date: Tue Mar 1 08:11:24 2022 New Revision: 1898497 URL: http://svn.apache.org/viewvc?rev=1898497=rev Log: Publish document 0.10.1 Modified: zeppelin/site/assets.html zeppelin/site/atom.xml zeppelin/site/community.html zeppelin/site/contribution/contributions.html zeppelin/site/contribution/documentation.html zeppelin/site/contribution/webapplication.html zeppelin/site/contribution/zeppelinweb/goodPracticeGuide01.html zeppelin/site/contribution/zeppelinweb/goodPracticeGuide02.html zeppelin/site/contribution/zeppelinweb/goodPracticeGuide03.html zeppelin/site/contribution/zeppelinweb/goodPracticeGuide04.html zeppelin/site/doap.rdf zeppelin/site/documentation.html zeppelin/site/download.html zeppelin/site/helium_packages.html zeppelin/site/index.html zeppelin/site/releases/zeppelin-release-0.5.0-incubating.html zeppelin/site/releases/zeppelin-release-0.5.5-incubating.html zeppelin/site/releases/zeppelin-release-0.5.6-incubating.html zeppelin/site/releases/zeppelin-release-0.6.0.html zeppelin/site/releases/zeppelin-release-0.6.1.html zeppelin/site/releases/zeppelin-release-0.6.2.html zeppelin/site/releases/zeppelin-release-0.7.0.html zeppelin/site/releases/zeppelin-release-0.7.1.html zeppelin/site/releases/zeppelin-release-0.7.2.html zeppelin/site/releases/zeppelin-release-0.7.3.html zeppelin/site/releases/zeppelin-release-0.8.0.html zeppelin/site/releases/zeppelin-release-0.8.1.html zeppelin/site/releases/zeppelin-release-0.8.2.html zeppelin/site/releases/zeppelin-release-0.9.0.html zeppelin/site/rss.xml zeppelin/site/screenshots.html zeppelin/site/sitemap.txt zeppelin/site/supported_interpreters.html Modified: zeppelin/site/assets.html URL: http://svn.apache.org/viewvc/zeppelin/site/assets.html?rev=1898497=1898496=1898497=diff == --- zeppelin/site/assets.html (original) +++ zeppelin/site/assets.html Tue Mar 1 08:11:24 2022 @@ -87,6 +87,7 @@ Docs Release +0.10.1 0.10.0 0.9.0 0.8.2 @@ -193,7 +194,7 @@ limitations under the License. - + Modified: zeppelin/site/atom.xml URL: http://svn.apache.org/viewvc/zeppelin/site/atom.xml?rev=1898497=1898496=1898497=diff == --- zeppelin/site/atom.xml (original) +++ zeppelin/site/atom.xml Tue Mar 1 08:11:24 2022 @@ -4,7 +4,7 @@ Apache Zeppelin http://zeppelin.apache.org/; rel="self"/> http://zeppelin.apache.org"/> - 2021-08-24T23:50:33+08:00 + 2022-03-01T08:08:28+00:00 http://zeppelin.apache.org The Apache Software Foundation Modified: zeppelin/site/community.html URL: http://svn.apache.org/viewvc/zeppelin/site/community.html?rev=1898497=1898496=1898497=diff == --- zeppelin/site/community.html (original) +++ zeppelin/site/community.html Tue Mar 1 08:11:24 2022 @@ -87,6 +87,7 @@ Docs Release +0.10.1 0.10.0 0.9.0 0.8.2 @@ -170,7 +171,7 @@ And help is always welcome! Issue Tracker Apache Zeppelin uses JIRA as an https://issues.apache.org/jira/browse/ZEPPELIN;>Issue Tracker. -Donât hesitate to report new bugs and improvements ideas. +Dont hesitate to report new bugs and improvements ideas. Contribute to the source code @@ -180,7 +181,7 @@ Donât hesitate to report new bugs a you important information about our review process, and pointers on how to make a good code contribution. You can visit our https://issues.apache.org/jira/browse/ZEPPELIN;>Issue Tracker to find issues to resolve, -and if your are a newcomer and donât know where to get started, we have set some https://issues.apache.org/jira/browse/ZEPPELIN-1245?jql=project%20%3D%20ZEPPELIN%20AND%20status%20%3D%20Open%20AND%20labels%20%3D%20beginner;>beginner issues. +and if your are a newcomer and dont know where to get started, we have set some https://issues.apache.org/jira/browse/ZEPPELIN-1245?jql=project%20%3D%20ZEPPELIN%20AND%20status%20%3D%20Open%20AND%20labels%20%3D%20beginner;>beginner issues. Other contributions @@ -265,7 +266,7 @@ for commit messages and patches. - + Modified: zeppelin/site/contribution/contributions.html URL: http://svn.apache.org/viewvc/zeppelin/site/contribution/contributions.html?rev=1898497=1898496=1898497=diff == --- zeppelin/site/contribution/contributions.html (original) +++ zeppelin/site/contribution/contributions.html Tue Mar 1 08:11:24 2022 @@ -87,6 +87,7 @@ Docs
[zeppelin] tag v0.10.1 created (now fd74504)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to tag v0.10.1 in repository https://gitbox.apache.org/repos/asf/zeppelin.git. at fd74504 (commit) No new revisions were added by this update.
[zeppelin] tag v0.10.1-docker deleted (was fd74504)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to tag v0.10.1-docker in repository https://gitbox.apache.org/repos/asf/zeppelin.git. *** WARNING: tag v0.10.1-docker was deleted! *** was fd74504 [MINOR] Update version in Dockerfile for 0.10.1 The revisions that were on this tag are still contained in other references; therefore, this change does not discard any commits from the repository.
svn commit: r52759 - /dev/zeppelin/zeppelin-0.10.1-rc2/ /release/zeppelin/zeppelin-0.10.1/
Author: zjffdu Date: Mon Feb 28 05:07:34 2022 New Revision: 52759 Log: Move 0.10.1 to release folder Added: release/zeppelin/zeppelin-0.10.1/ - copied from r52758, dev/zeppelin/zeppelin-0.10.1-rc2/ Removed: dev/zeppelin/zeppelin-0.10.1-rc2/
[zeppelin] tag 0.10.1-docker deleted (was fd74504)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to tag 0.10.1-docker in repository https://gitbox.apache.org/repos/asf/zeppelin.git. *** WARNING: tag 0.10.1-docker was deleted! *** was fd74504 [MINOR] Update version in Dockerfile for 0.10.1 The revisions that were on this tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[zeppelin] tag v0.10.1-docker created (now fd74504)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to tag v0.10.1-docker in repository https://gitbox.apache.org/repos/asf/zeppelin.git. at fd74504 (commit) No new revisions were added by this update.
[zeppelin] tag 0.10.1-docker created (now fd74504)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to tag 0.10.1-docker in repository https://gitbox.apache.org/repos/asf/zeppelin.git. at fd74504 (commit) No new revisions were added by this update.
[zeppelin] branch branch-0.10 updated: [MINOR] Update version in Dockerfile for 0.10.1
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new fd74504 [MINOR] Update version in Dockerfile for 0.10.1 fd74504 is described below commit fd74504199247f8a9f9c3950abf193c97cf1d771 Author: Jeff Zhang AuthorDate: Mon Feb 28 12:08:17 2022 +0800 [MINOR] Update version in Dockerfile for 0.10.1 --- scripts/docker/zeppelin/bin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/docker/zeppelin/bin/Dockerfile b/scripts/docker/zeppelin/bin/Dockerfile index 66d794e..8deb91c 100644 --- a/scripts/docker/zeppelin/bin/Dockerfile +++ b/scripts/docker/zeppelin/bin/Dockerfile @@ -17,7 +17,7 @@ FROM ubuntu:20.04 LABEL maintainer="Apache Software Foundation " -ENV Z_VERSION="0.10.2-SNAPSHOT" +ENV Z_VERSION="0.10.1" ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \ ZEPPELIN_HOME="/opt/zeppelin" \
[zeppelin] branch master updated: [ZEPPELIN-5639] Support node label in yarn interpreter launcher
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new baae6ed [ZEPPELIN-5639] Support node label in yarn interpreter launcher baae6ed is described below commit baae6edfd296352dcc2c6efa69d18e24e0ae743a Author: huage1994 AuthorDate: Fri Jan 28 22:54:14 2022 +0800 [ZEPPELIN-5639] Support node label in yarn interpreter launcher ### What is this PR for? Support node label in yarn interpreter launcher, so that user can run interpreter in specific node. ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? * Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN-5639 ### How should this be tested? CI passed ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: huage1994 Closes #4293 from huage1994/ZEPPELIN-5639 and squashes the following commits: 7c5dfb3a98 [huage1994] [ZEPPELIN-5639] Support node label in yarn interpreter launcher --- docs/quickstart/yarn.md | 5 + .../zeppelin/interpreter/launcher/YarnRemoteInterpreterProcess.java | 4 2 files changed, 9 insertions(+) diff --git a/docs/quickstart/yarn.md b/docs/quickstart/yarn.md index 60fb48e..0aee85c 100644 --- a/docs/quickstart/yarn.md +++ b/docs/quickstart/yarn.md @@ -64,6 +64,11 @@ Besides that, you can also specify other properties as following table. default yarn queue name + +zeppelin.interpreter.yarn.node.label.expression + +yarn node label expression specified for interpreter process + ## Differences with non-yarn interpreter mode (local mode) diff --git a/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnRemoteInterpreterProcess.java b/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnRemoteInterpreterProcess.java index 33bca6c..31e1f74 100644 --- a/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnRemoteInterpreterProcess.java +++ b/zeppelin-plugins/launcher/yarn/src/main/java/org/apache/zeppelin/interpreter/launcher/YarnRemoteInterpreterProcess.java @@ -178,6 +178,10 @@ public class YarnRemoteInterpreterProcess extends RemoteInterpreterProcess { this.appId = appResponse.getApplicationId(); ApplicationSubmissionContext appContext = yarnApplication.getApplicationSubmissionContext(); appContext = createApplicationSubmissionContext(appContext); + String YARN_NODE_LABEL_EXPRESSION = "zeppelin.interpreter.yarn.node.label.expression"; + if(StringUtils.isNotEmpty(properties.getProperty(YARN_NODE_LABEL_EXPRESSION,""))) { + appContext.setNodeLabelExpression(properties.getProperty(YARN_NODE_LABEL_EXPRESSION)); + } yarnClient.submitApplication(appContext); long start = System.currentTimeMillis();
[zeppelin] branch master updated: [ZEPPELIN-5654] Implement GCSNotebookRepo Move
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new aa32f62 [ZEPPELIN-5654] Implement GCSNotebookRepo Move aa32f62 is described below commit aa32f6258d8df01def4b2552b244f0071f54a24e Author: Bagus Nurtomo AuthorDate: Sat Feb 19 15:46:35 2022 +0700 [ZEPPELIN-5654] Implement GCSNotebookRepo Move ### What is this PR for? This is to implement move function that is used for renaming notebooks and moving it to trash. Currently it would result in a no op, making users unable to rename notebooks if using GCS as the notebook repository. ### What type of PR is it? Feature ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-5654 ### How should this be tested? CI ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No? Author: Bagus Nurtomo Closes #4297 from BagusThanatos/implement-move-gcsnotebookrepo and squashes the following commits: ebcc8870a3 [Bagus Nurtomo] Remove unused variable 102a8ca4ec [Bagus Nurtomo] Fix Tests 65e7c3e9d5 [Bagus Nurtomo] Add test 4ca5009291 [Bagus Nurtomo] Implement GCSNotebookRepo move --- .../apache/zeppelin/notebook/repo/GCSNotebookRepo.java | 12 ++-- .../zeppelin/notebook/repo/GCSNotebookRepoTest.java | 16 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java index 529225a..0f2aed3 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/main/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepo.java @@ -201,8 +201,16 @@ public class GCSNotebookRepo implements NotebookRepo { } @Override - public void move(String noteId, String notePath, String newNotePath, AuthenticationInfo subject) { - + public void move(String noteId, String notePath, String newNotePath, AuthenticationInfo subject) throws IOException { +Preconditions.checkArgument(StringUtils.isNotEmpty(noteId)); +BlobId sourceBlobId = makeBlobId(noteId, notePath); +BlobId destinationBlobId = makeBlobId(noteId, newNotePath); +try { + storage.get(sourceBlobId).copyTo(destinationBlobId); +} catch (Exception se) { + throw new IOException("Could not copy from " + sourceBlobId.toString() + " to " + destinationBlobId.toString() + ": " + se.getMessage(), se); +} +remove(noteId, notePath, subject); } @Override diff --git a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java index 295305b..69ef76f 100644 --- a/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java +++ b/zeppelin-plugins/notebookrepo/gcs/src/test/java/org/apache/zeppelin/notebook/repo/GCSNotebookRepoTest.java @@ -96,6 +96,7 @@ public class GCSNotebookRepoTest { Paragraph p = new Paragraph(note, null); p.setText("text"); p.setStatus(Status.RUNNING); +p.setAuthenticationInfo(new AuthenticationInfo("anonymous", (String)null, "anonymous")); note.addParagraph(p); return note; } @@ -186,6 +187,21 @@ public class GCSNotebookRepoTest { assertThat(storage.get(makeBlobId(runningNote.getId(), runningNote.getPath(.isNull(); } + @Test + public void testMove_nonexistent() throws Exception { +try { + notebookRepo.move("id", "/name", "/name_new", AUTH_INFO); + fail(); +} catch (IOException e) {} + } + + @Test + public void testMove() throws Exception { +create(runningNote); +notebookRepo.move(runningNote.getId(), runningNote.getPath(), runningNote.getPath() + "_new", AUTH_INFO); +assertThat(storage.get(makeBlobId(runningNote.getId(), runningNote.getPath(.isNull(); + } + private String makeName(String relativePath) { if (basePath.isPresent()) { return basePath.get() + "/" + relativePath;
[zeppelin] branch master updated: [ZEPPELIN-5629] spark.driver.extraJavaOptions setting in %spark.conf does not work well
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new b870ee6 [ZEPPELIN-5629] spark.driver.extraJavaOptions setting in %spark.conf does not work well b870ee6 is described below commit b870ee69a0751d2eeefe43cfb22fa1acc5ef8a12 Author: huage1994 AuthorDate: Mon Jan 17 18:08:24 2022 +0800 [ZEPPELIN-5629] spark.driver.extraJavaOptions setting in %spark.conf does not work well ### What is this PR for? The value of `spark.driver.extraJavaOptions` setting in `%spark.conf` will override the its initial value provided by zeppelin. It should append instead of override. This PR fix this bug. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? * Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN-5629 ### How should this be tested? * CI passed ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: huage1994 Closes #4289 from huage1994/ZEPPELIN-5629 and squashes the following commits: 24805331fb [huage1994] [ZEPPELIN-5629] spark.driver.extraJavaOptions setting in %spark.conf does not work well --- bin/interpreter.sh | 4 ++-- .../interpreter/launcher/SparkInterpreterLauncher.java | 16 ++-- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index 6d9c048..baeaa5b 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -300,9 +300,9 @@ if [[ -n "${SPARK_SUBMIT}" ]]; then IFS=' ' read -r -a SPARK_SUBMIT_OPTIONS_ARRAY <<< "${SPARK_SUBMIT_OPTIONS}" IFS='|' read -r -a ZEPPELIN_SPARK_CONF_ARRAY <<< "${ZEPPELIN_SPARK_CONF}" if [[ "${ZEPPELIN_SPARK_YARN_CLUSTER}" == "true" ]]; then -INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}" "--driver-java-options" "${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}" "${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") +INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}" "--driver-java-options" "${SPARK_DRIVER_EXTRAJAVAOPTIONS_CONF} ${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}" "${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") else -INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}" "--driver-class-path" "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}" "--driver-java-options" "${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}" "${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") +INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}" "--driver-class-path" "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}" "--driver-java-options" "${SPARK_DRIVER_EXTRAJAVAOPTIONS_CONF} ${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}" "${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") fi elif [[ -n "${ZEPPELIN_FLINK_APPLICATION_MODE}" ]]; then IFS='|' read -r -a ZEPPELIN_FLINK_APPLICATION_MODE_CONF_ARRAY <<< "${ZEPPELIN_FLINK_APPLICATION_MODE_CONF}" diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java index 239976d..02f481d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java @@ -27,10 +27,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.uti
svn commit: r52719 - in /dev/zeppelin/zeppelin-0.10.1-rc2: zeppelin-0.10.1-bin-netinst.tgz zeppelin-0.10.1-bin-netinst.tgz.asc zeppelin-0.10.1-bin-netinst.tgz.sha512 zeppelin-0.10.1.tgz zeppelin-0.10.
Author: zjffdu Date: Thu Feb 24 05:32:03 2022 New Revision: 52719 Log: Upload 0.10.1 rc2 Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz (with props) dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz.asc dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz.sha512 dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz (with props) dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz.asc dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz.sha512 Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz == Binary file - no diff available. Propchange: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz -- svn:mime-type = application/octet-stream Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz.asc == --- dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz.asc (added) +++ dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz.asc Thu Feb 24 05:32:03 2022 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCAAdFiEE9k6XRfAhCMKdxRepCQ++FNmxfR8FAmIXB70ACgkQCQ++FNmx +fR9OAw/9HaZAePVjzhz9nO+MaSi2rRpwzdrXzd81vk6sqY30y2bpo+fj+BkD6nwu +/qQC26kwbOtBWjtkJFDMqnztEzfTNR4/4FlJon8xzrnfgbePieOoXOGHL8SHSAML +wyxXRMORfDdZV0Wqy4LUNdgG4oI9LsrpvNxEUV0PpkkbolwJUzxl1qdNyiUUCO8n +Rw6hLPApYLm7tyu2KW04DIfBlpPad7XSXZeWbOrRgOXiR5JuBepkJlzAz24sPnwo +qi6iXwHNFKKs7VHzh5w60zgZ/C1FYAOypdv8ioOxiZ7kWWEaoVLLcqSB8yxeUtTY +SKY1pixSKVEViML73yjmJJT4EI94Ow2enPtjWy4yOJkjiUMl3fIMVIDuC89q1THw +ebOEJ0EqVbAnmuqrTpLLPWLZXOHdoUd/ZJDfD6E3uwTWjsWwwoNJ62J2WQRhR76+ +8ZYQgVRV2o2fDaSh4364nr7PvM7finIrg/MJAh1mn0R7YN0EDpEnxqawKMj6QKAP +lioo0YW+Jn73H3N60+Y97BsmC9wkDM79PUP0AeKhXyn538w4LvicALVUD4ArhjIY +93OJ2wq6N7a1ubs9iVIFuo9xqdFemdNl0i9D6Yd37q1BUFd55BMuvbsX+bB315Pw +sgQfBd002rf3m2NTJH1jPPvhZA0wZIMMp40hPGPlBQ2/lNmX1B4= +=lBGv +-END PGP SIGNATURE- Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz.sha512 == --- dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz.sha512 (added) +++ dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-netinst.tgz.sha512 Thu Feb 24 05:32:03 2022 @@ -0,0 +1 @@ +59d7261adfd3b17b6648e083cc2e06cfb0c5aedd7e76498175073a80c224c6bc05ab562739b2fe8f0db566d01b99203a5fb09b0e680916b046ae58f3146695a9 zeppelin-0.10.1-bin-netinst.tgz Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz == Binary file - no diff available. Propchange: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz -- svn:mime-type = application/octet-stream Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz.asc == --- dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz.asc (added) +++ dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz.asc Thu Feb 24 05:32:03 2022 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCAAdFiEE9k6XRfAhCMKdxRepCQ++FNmxfR8FAmIXB5cACgkQCQ++FNmx +fR99hxAAr0yaydRDhKf4cRaXYwFCqEPN+sUUn7KqKpaeRfgF1GHbpgrJzNWKRmYn +laMMlRLrPKk0YE6+vQb3O4EFpVIivmk9QwSr0uwIo9jo1htFFVXtOcp42NAnjQZX +E30rQjYgzH3PFSQQLZTl3GPFYVKallb28YnclLRo3YDB1yQAzB02LIh7KgF11Lhy +5OgLniSqXZImki5MDz6+RNICxGWOaHs1keMJWp856ZdTcxMBAiq/hSCO4sEaj3iZ +ytvBpRzvUFmwcQF3uxZDcAtvsX9VQviY03Q/KieJKZ8YItE80I6kNSIHB2E2VWZy +GAM3Bno/WGLGv3e3U3cNheGLZb+bKnjQh7wCEXiQxcM+X8dGquAmPSPWrgwvS7np +q5N2pt6hNUjXmhP/qxmDDKTUK0Li040stZLToQOeEvflhUmDsnve7RxGE9hYXC1n +MPJhL5Hq6lS9ZAMnNlB/S5pFwmAfJME+pbEqp9R6YKS66+lEnWZXf0SjBmNmjxMY +cOLQ1Qp8Mkhj60dFPMKSV7fRn+nNdVzid4oN6nkH2dd5Yoq4rOjTL8K479588ssg +0wb2Ow9Dgy7baonxMMZrUF/j2+9QnGkP2WfZMESJYGZ9Qz0FkNcnNuwKSJcMjnyq +86NJ8pGVw8SUrlFRbMeTfcqpKUcHuyl0a2O9mrLCuNqw3PTzDIY= +=CD5h +-END PGP SIGNATURE- Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz.sha512 == --- dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz.sha512 (added) +++ dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1.tgz.sha512 Thu Feb 24 05:32:03 2022 @@ -0,0 +1 @@ +fb49fdffcb54bb0acbaac9ade090afd75abf8226bf2c2c998423fd386790a11e85a4e3cf8885a0f3eac42fe49406616ada2d379da38b207a2e3a112048e0f67c zeppelin-0.10.1.tgz
svn commit: r52718 - in /dev/zeppelin/zeppelin-0.10.1-rc2: zeppelin-0.10.1-bin-all.tgz zeppelin-0.10.1-bin-all.tgz.asc zeppelin-0.10.1-bin-all.tgz.sha512
Author: zjffdu Date: Thu Feb 24 05:27:07 2022 New Revision: 52718 Log: Upload 0.10.1 rc2 bin all Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz (with props) dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz.asc dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz.sha512 Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz == Binary file - no diff available. Propchange: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz -- svn:mime-type = application/octet-stream Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz.asc == --- dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz.asc (added) +++ dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz.asc Thu Feb 24 05:27:07 2022 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCAAdFiEE9k6XRfAhCMKdxRepCQ++FNmxfR8FAmIXBoIACgkQCQ++FNmx +fR8gEg//X7tFrewh5u8Ss4JwdGFCCsThYcbBog6B5vV/vKi+5rmVmfn1kcXlpByP +aPj3C3OpNqYjex6Cjz0mgli1HUXyOIg1Pmi8nsSkCQr2icpRyAGS4Gzba55tKvbi +GSCqW473sMiriDTa7h5rQW5zRytwjdZpN1p7B+KIYdcrhGJ0REzVMt2Z587prIPp +s+z1lbyrr8lmlVhdGXTa87UAiIfktKeqRYvPckrzHUfLJlmMQpoKOwMeMmxYxVDl +QnLHprBzhz+iQG/udAnjJgwhPgNCADUH5EiDrQkmsr7fGDLaVsPxhfbGZQ3OdPLb +4N2s+HS0frZAljJe6eHTxzPwIqvxy2epxZH6lplWIQ3ao73wPpSjy5AG+lTc/nSz +Ymi08NpmtjloWFI5JW8glJaCHt5LtKOv15pbjQ5lKL/x9BjXJ1ve9TcZhT+ESz57 +wn9O9+1UBOiF6cP6SlJTkWakd3jy7fJMNXqB5L/atSemSStRAXjXznqnHrOlSsim +ShaUmRSkVs0lDRSIg4Ug2o7Wh/C+XLIOWB0IGpT0P0Nw9AxlrCw/VM6o2jV9SJDS +H675X7hWQixizRVOOBwjSiColE9J85G6C5NLxQ/I7U+LFthU48HG5eWOxebbTP4x +uTxKOm7OWbRX4oHXgviDQhTRHU12OotX2cqciW21i6RcQ1caS64= +=nggu +-END PGP SIGNATURE- Added: dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz.sha512 == --- dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz.sha512 (added) +++ dev/zeppelin/zeppelin-0.10.1-rc2/zeppelin-0.10.1-bin-all.tgz.sha512 Thu Feb 24 05:27:07 2022 @@ -0,0 +1 @@ +94cc52d3121afecfd12e1061bb6f4e7e227629236e4374d2fad1e49403a52b31afc3ff34216e5f3bd9b95324ac6e586da4131712b84aefac0a01839404aa zeppelin-0.10.1-bin-all.tgz
svn commit: r52717 - /dev/zeppelin/zeppelin-0.10.1-rc2/
Author: zjffdu Date: Thu Feb 24 05:18:34 2022 New Revision: 52717 Log: Create folder for 0.10.1 rc2 Added: dev/zeppelin/zeppelin-0.10.1-rc2/
[zeppelin] branch master updated: [ZEPPELIN-5576] Improve Zeppelin CI
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new d8d18f5 [ZEPPELIN-5576] Improve Zeppelin CI d8d18f5 is described below commit d8d18f59b2926c90b34b5cfb27a3c2c3174f62a5 Author: Jeff Zhang AuthorDate: Tue Feb 22 09:57:42 2022 +0100 [ZEPPELIN-5576] Improve Zeppelin CI ### What is this PR for? Restructure the CI in `core.yaml` * Split interpreters CI into 5 groups: * jupyter-interpreter, rlang, python * non-core interpreters (sh, markdown, etc) * spark * flink * livy * Run different profiles' spark test in one bulid. Only run it under different python versions. * Refine the hadoop2 and hadoop3 profiles, use different hadoop profiles in integration test (`SparkIntegrationTest`, `ZeppelinSparkClusterTest`) ### What type of PR is it? [Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5576 ### How should this be tested? * CI pass except the frontend e2e test (not related) ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? NO * Is there breaking changes for older versions? NO * Does this needs documentation? NO Author: Jeff Zhang Author: Philipp Dallig Closes #4266 from zjffdu/ZEPPELIN-5576 and squashes the following commits: 1010d043d4 [Philipp Dallig] Adjust interpreter pom.xml 76117056b3 [Jeff Zhang] address comments e5477a2a80 [Jeff Zhang] update on mamba 6fab322589 [Jeff Zhang] use mamba ed7b431bed [Jeff Zhang] Remove hadoop-client-api in zeppelin-interpreter 8eca79545b [Jeff Zhang] address comment 458040cadf [Jeff Zhang] address comments 7c4ca219da [Jeff Zhang] address comments 19c9f89300 [Jeff Zhang] address comment 6ce636853d [Jeff Zhang] address comments 916d86ebfc [Jeff Zhang] address comment 25cdd68b0d [Jeff Zhang] use mmaba cba304f45b [Jeff Zhang] update 175d75e944 [Jeff Zhang] [ZEPPELIN-5576] Improve Zeppelin CI --- .github/workflows/core.yml | 297 +--- .github/workflows/frontend.yml | 7 + alluxio/pom.xml| 7 + pom.xml| 505 ++--- rlang/pom.xml | 15 +- scalding/pom.xml | 6 + spark/interpreter/pom.xml | 15 +- .../zeppelin/spark/SparkIRInterpreterTest.java | 84 ++-- .../zeppelin/spark/SparkRInterpreterTest.java | 3 +- spark/pom.xml | 1 - spark/spark-dependencies/pom.xml | 20 +- spark/spark-scala-parent/pom.xml | 15 + spark/spark-shims/pom.xml | 2 +- testing/env_python_3_with_flink_110.yml| 27 -- testing/env_python_3_with_flink_111.yml| 28 -- zeppelin-interpreter-integration/pom.xml | 140 -- .../zeppelin/integration/FlinkIntegrationTest.java | 6 +- .../zeppelin/integration/MiniHadoopCluster.java| 4 +- .../zeppelin/integration/SparkIntegrationTest.java | 19 + .../integration/SparkIntegrationTest24.java| 4 +- .../integration/ZeppelinSparkClusterTest.java | 109 +++-- zeppelin-interpreter/pom.xml | 61 ++- zeppelin-plugins/launcher/yarn/pom.xml | 68 ++- zeppelin-plugins/notebookrepo/filesystem/pom.xml | 59 ++- zeppelin-server/pom.xml| 59 +-- zeppelin-zengine/pom.xml | 56 ++- 26 files changed, 1095 insertions(+), 522 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index b6426284..808356d 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -26,13 +26,14 @@ defaults: shell: bash -l {0} jobs: - test-core-modules: + # test on core-modules (zeppelin-interpreter,zeppelin-zengine,zeppelin-server), + # some interpreters are included, because zeppelin-server test depends on them: spark, shell & markdown + core-modules: runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: hadoop: [hadoop2, hadoop3] -python: [3.7, 3.8] steps: - name: Checkout uses: actions/checkout@v2 @@ -52,29 +53,34 @@ jobs: key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-zeppelin- - - name: Setup conda environment with python ${{ matrix.python }} and R + - name: install application with some interpreter +run: ./mvnw install -Pb
[zeppelin] branch branch-0.10 updated (c99e75c -> 72dbfee)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git. from c99e75c [HOTFIX] Fix CI failure add 1654ce3 Preparing Apache Zeppelin release 0.10.1 new 72dbfee Preparing development version 0.10.2-SNAPSHOT The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes:
[zeppelin] 01/01: Preparing development version 0.10.2-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git commit 72dbfee6bf463d2fe8909035452c525a95381c05 Author: Jeff Zhang AuthorDate: Tue Feb 22 20:17:06 2022 +0800 Preparing development version 0.10.2-SNAPSHOT --- alluxio/pom.xml | 2 +- angular/pom.xml | 2 +- beam/pom.xml| 2 +- bigquery/pom.xml| 2 +- cassandra/pom.xml | 2 +- elasticsearch/pom.xml | 2 +- file/pom.xml| 2 +- flink-cmd/pom.xml | 2 +- flink/flink-scala-2.11/pom.xml | 4 ++-- flink/flink-scala-2.12/pom.xml | 4 ++-- flink/flink-scala-parent/pom.xml| 4 ++-- flink/flink-shims/pom.xml | 2 +- flink/flink1.10-shims/pom.xml | 2 +- flink/flink1.11-shims/pom.xml | 2 +- flink/flink1.12-shims/pom.xml | 2 +- flink/flink1.13-shims/pom.xml | 4 ++-- flink/flink1.14-shims/pom.xml | 4 ++-- flink/pom.xml | 2 +- geode/pom.xml | 2 +- groovy/pom.xml | 2 +- hazelcastjet/pom.xml| 2 +- hbase/pom.xml | 2 +- helium-dev/pom.xml | 2 +- ignite/pom.xml | 2 +- influxdb/pom.xml| 2 +- java/pom.xml| 2 +- jdbc/pom.xml| 2 +- kotlin/pom.xml | 2 +- ksql/pom.xml| 2 +- kylin/pom.xml | 2 +- lens/pom.xml| 2 +- livy/pom.xml| 2 +- markdown/pom.xml| 2 +- mongodb/pom.xml | 2 +- neo4j/pom.xml | 2 +- pig/pom.xml | 2 +- pom.xml | 2 +- python/pom.xml | 2 +- rlang/pom.xml | 2 +- sap/pom.xml | 2 +- scalding/pom.xml| 2 +- scio/pom.xml| 2 +- shell/pom.xml | 2 +- spark-submit/pom.xml| 2 +- spark/interpreter/pom.xml | 2 +- spark/pom.xml | 2 +- spark/scala-2.10/pom.xml| 2 +- spark/scala-2.11/pom.xml| 2 +- spark/scala-2.12/pom.xml| 2 +- spark/spark-dependencies/pom.xml| 2 +- spark/spark-scala-parent/pom.xml| 2 +- spark/spark-shims/pom.xml | 2 +- spark/spark1-shims/pom.xml | 2 +- spark/spark2-shims/pom.xml | 2 +- spark/spark3-shims/pom.xml | 2 +- sparql/pom.xml | 2 +- submarine/pom.xml | 6 +++--- zeppelin-client-examples/pom.xml| 2 +- zeppelin-client/pom.xml | 2 +- zeppelin-common/pom.xml | 2 +- zeppelin-display/pom.xml| 2 +- zeppelin-distribution/pom.xml | 2 +- zeppelin-examples/pom.xml | 2 +- zeppelin-examples/zeppelin-example-clock/pom.xml| 4 ++-- zeppelin-examples/zeppelin-example-horizontalbar/pom.xml| 4 ++-- zeppelin-examples/zeppelin-example-spell-echo/pom.xml | 4 ++-- zeppelin-examples/zeppelin-example-spell-flowchart/pom.xml | 4 ++-- zeppelin-examples/zeppelin-example-spell-markdown/pom.xml | 4
[zeppelin] tag v0.10.1-rc2 created (now 1654ce3)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to tag v0.10.1-rc2 in repository https://gitbox.apache.org/repos/asf/zeppelin.git. at 1654ce3 (commit) This tag includes the following new commits: new 1654ce3 Preparing Apache Zeppelin release 0.10.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[zeppelin] 01/01: Preparing Apache Zeppelin release 0.10.1
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to tag v0.10.1-rc2 in repository https://gitbox.apache.org/repos/asf/zeppelin.git commit 1654ce3847f4ebce7b228f54671340569d25a32e Author: Jeff Zhang AuthorDate: Tue Feb 22 20:17:00 2022 +0800 Preparing Apache Zeppelin release 0.10.1 --- alluxio/pom.xml | 2 +- angular/pom.xml | 2 +- beam/pom.xml| 2 +- bigquery/pom.xml| 2 +- cassandra/pom.xml | 2 +- elasticsearch/pom.xml | 2 +- file/pom.xml| 2 +- flink-cmd/pom.xml | 2 +- flink/flink-scala-2.11/pom.xml | 4 ++-- flink/flink-scala-2.12/pom.xml | 4 ++-- flink/flink-scala-parent/pom.xml| 4 ++-- flink/flink-shims/pom.xml | 2 +- flink/flink1.10-shims/pom.xml | 2 +- flink/flink1.11-shims/pom.xml | 2 +- flink/flink1.12-shims/pom.xml | 2 +- flink/flink1.13-shims/pom.xml | 4 ++-- flink/flink1.14-shims/pom.xml | 4 ++-- flink/pom.xml | 2 +- geode/pom.xml | 2 +- groovy/pom.xml | 2 +- hazelcastjet/pom.xml| 2 +- hbase/pom.xml | 2 +- helium-dev/pom.xml | 2 +- ignite/pom.xml | 2 +- influxdb/pom.xml| 2 +- java/pom.xml| 2 +- jdbc/pom.xml| 2 +- kotlin/pom.xml | 2 +- ksql/pom.xml| 2 +- kylin/pom.xml | 2 +- lens/pom.xml| 2 +- livy/pom.xml| 2 +- markdown/pom.xml| 2 +- mongodb/pom.xml | 2 +- neo4j/pom.xml | 2 +- pig/pom.xml | 2 +- pom.xml | 2 +- python/pom.xml | 2 +- rlang/pom.xml | 2 +- sap/pom.xml | 2 +- scalding/pom.xml| 2 +- scio/pom.xml| 2 +- shell/pom.xml | 2 +- spark-submit/pom.xml| 2 +- spark/interpreter/pom.xml | 2 +- spark/pom.xml | 2 +- spark/scala-2.10/pom.xml| 2 +- spark/scala-2.11/pom.xml| 2 +- spark/scala-2.12/pom.xml| 2 +- spark/spark-dependencies/pom.xml| 2 +- spark/spark-scala-parent/pom.xml| 2 +- spark/spark-shims/pom.xml | 2 +- spark/spark1-shims/pom.xml | 2 +- spark/spark2-shims/pom.xml | 2 +- spark/spark3-shims/pom.xml | 2 +- sparql/pom.xml | 2 +- submarine/pom.xml | 6 +++--- zeppelin-client-examples/pom.xml| 2 +- zeppelin-client/pom.xml | 2 +- zeppelin-common/pom.xml | 2 +- zeppelin-display/pom.xml| 2 +- zeppelin-distribution/pom.xml | 2 +- zeppelin-examples/pom.xml | 2 +- zeppelin-examples/zeppelin-example-clock/pom.xml| 4 ++-- zeppelin-examples/zeppelin-example-horizontalbar/pom.xml| 4 ++-- zeppelin-examples/zeppelin-example-spell-echo/pom.xml | 4 ++-- zeppelin-examples/zeppelin-example-spell-flowchart/pom.xml | 4 ++-- zeppelin-examples/zeppelin-example-spell-markdown/pom.xml | 4 ++-- zeppelin
[zeppelin] branch branch-0.10 updated (7775eb3 -> c99e75c)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git. from 7775eb3 [ZEPPELIN-5625] Potentially misleading error message when sparkR backend is not available new 88bf9b1 [ZEPPELIN-5604] Interpreter scheduler may not be shutdown properly new c99e75c [HOTFIX] Fix CI failure The 2 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .github/workflows/core.yml | 50 +++--- .../apache/zeppelin/spark/SparkInterpreter.java| 10 ++--- .../interpreter/ManagedInterpreterGroup.java | 29 ++--- .../launcher/SparkInterpreterLauncher.java | 1 + 4 files changed, 64 insertions(+), 26 deletions(-)
[zeppelin] 01/02: [ZEPPELIN-5604] Interpreter scheduler may not be shutdown properly
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git commit 88bf9b135ee858d1b929ac663f6381374e7e6e70 Author: Jeff Zhang AuthorDate: Tue Dec 7 23:56:50 2021 +0800 [ZEPPELIN-5604] Interpreter scheduler may not be shutdown properly --- .github/workflows/core.yml | 21 +--- .../interpreter/ManagedInterpreterGroup.java | 29 +++--- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 29c2a0b..6757179 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -62,6 +62,8 @@ jobs: channel-priority: strict - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" conda list conda info @@ -103,6 +105,8 @@ jobs: auto-activate-base: false - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: verify interpreter run: mvn verify -DskipRat -am -pl .,zeppelin-interpreter,zeppelin-interpreter-shaded,${INTERPRETERS} -Pscala-2.10 -B @@ -136,6 +140,8 @@ jobs: auto-activate-base: false - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -211,6 +217,8 @@ jobs: auto-activate-base: false - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -252,6 +260,8 @@ jobs: auto-activate-base: false - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -290,6 +300,8 @@ jobs: auto-activate-base: false - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -331,6 +343,8 @@ jobs: auto-activate-base: false - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -372,6 +386,8 @@ jobs: auto-activate-base: false - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B @@ -412,6 +428,8 @@ jobs: auto-activate-base: false - name: Make IRkernel available to Jupyter run: | + R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" + R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.2 -Pspark-scala-2.12 -Phadoop2 -B @@ -446,9 +464,6 @@ jobs: environment-file: testing/env_python_3_with_R.yml python-version: 3.7 auto-activate-base: false - - name: Make IRkernel available to Jupyter -run: | - R -e "IRkernel::installspec()
[zeppelin] 02/02: [HOTFIX] Fix CI failure
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git commit c99e75cdc1a1e141bb83caedd1bd4d817ee07840 Author: Jeff Zhang AuthorDate: Fri Feb 18 13:38:15 2022 +0800 [HOTFIX] Fix CI failure --- .github/workflows/core.yml | 65 +++--- .../apache/zeppelin/spark/SparkInterpreter.java| 10 ++-- .../launcher/SparkInterpreterLauncher.java | 1 + 3 files changed, 50 insertions(+), 26 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 6757179..21a355d 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -58,15 +58,14 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_${{ matrix.python }}_with_R.yml python-version: ${{ matrix.python }} + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false - channel-priority: strict + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - conda list - conda info - name: install application with some interpreter run: mvn install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -B - name: install and test plugins @@ -102,11 +101,13 @@ jobs: activate-environment: python_3_with_R_and_tensorflow environment-file: testing/env_python_3_with_R_and_tensorflow.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: verify interpreter run: mvn verify -DskipRat -am -pl .,zeppelin-interpreter,zeppelin-interpreter-shaded,${INTERPRETERS} -Pscala-2.10 -B @@ -137,11 +138,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_3_with_R.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -180,7 +183,11 @@ jobs: activate-environment: python_3_with_flink environment-file: testing/env_python_3_with_flink_${{ matrix.flink }}.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: install environment run: | mvn install -DskipTests -DskipRat -am -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -B @@ -214,11 +221,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_3_with_R.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -257,11 +266,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_3_with_R.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name:
[zeppelin] branch master updated: [ZEPPELIN-5642] Provide REST API and SDK to get noteId by notePath
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 631d6f1 [ZEPPELIN-5642] Provide REST API and SDK to get noteId by notePath 631d6f1 is described below commit 631d6f18108628c8014261843b1017ac8e6a72fd Author: huage1994 AuthorDate: Wed Jan 26 16:37:55 2022 +0800 [ZEPPELIN-5642] Provide REST API and SDK to get noteId by notePath ### What is this PR for? Provide REST API and SDK to get noteId by notePath. ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? * Open an issue on Jira https://issues.apache.org/jira/browse/ZEPPELIN-5642 ### How should this be tested? CI passed ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: huage1994 Closes #4292 from huage1994/ZEPPELIN-5642 and squashes the following commits: 6fbebfb0fd [huage1994] [ZEPPELIN-5642] Provide REST API and SDK to get noteId by notePath --- .../org/apache/zeppelin/client/ZeppelinClient.java | 25 + .../org/apache/zeppelin/rest/NotebookRestApi.java | 34 + .../rest/message/GetNoteByPathRequest.java | 32 .../apache/zeppelin/service/NotebookService.java | 25 + .../apache/zeppelin/rest/NotebookRestApiTest.java | 43 ++ .../org/apache/zeppelin/notebook/NoteManager.java | 5 +++ .../org/apache/zeppelin/notebook/Notebook.java | 4 ++ 7 files changed, 160 insertions(+), 8 deletions(-) diff --git a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java index 9daec97..2f8f6bc 100644 --- a/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java +++ b/zeppelin-client/src/main/java/org/apache/zeppelin/client/ZeppelinClient.java @@ -382,6 +382,27 @@ public class ZeppelinClient { .get("/notebook/{noteId}") .routeParam("noteId", noteId) .asJson(); +return extractNoteResultFromResponse(response); + } + + /** + * Query {@link NoteResult} with provided notePath. + * + * @param notePath + * @return + * @throws Exception + */ + public NoteResult queryNoteResultByPath(String notePath) throws Exception { +JSONObject bodyObject = new JSONObject(); +bodyObject.put("notePath", notePath); +HttpResponse response = Unirest +.post("/notebook/getByPath") +.body(bodyObject) +.asJson(); +return extractNoteResultFromResponse(response); + } + + private NoteResult extractNoteResultFromResponse(HttpResponse response) throws Exception { checkResponse(response); JsonNode jsonNode = response.getBody(); checkJsonNodeStatus(jsonNode); @@ -394,6 +415,10 @@ public class ZeppelinClient { isRunning = Boolean.parseBoolean(infoJsonObject.getString("isRunning")); } } +String noteId = null; +if (noteJsonObject.has("id")) { + noteId = noteJsonObject.getString("id"); +} String notePath = null; if (noteJsonObject.has("path")) { diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index a4b2004..d68a68a 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -53,14 +53,7 @@ import org.apache.zeppelin.rest.exception.BadRequestException; import org.apache.zeppelin.rest.exception.ForbiddenException; import org.apache.zeppelin.rest.exception.NoteNotFoundException; import org.apache.zeppelin.rest.exception.ParagraphNotFoundException; -import org.apache.zeppelin.rest.message.CronRequest; -import org.apache.zeppelin.rest.message.NewNoteRequest; -import org.apache.zeppelin.rest.message.NewParagraphRequest; -import org.apache.zeppelin.rest.message.NoteJobStatus; -import org.apache.zeppelin.rest.message.ParagraphJobStatus; -import org.apache.zeppelin.rest.message.RenameNoteRequest; -import org.apache.zeppelin.rest.message.ParametersRequest; -import org.apache.zeppelin.rest.message.UpdateParagraphRequest; +import org.apache.zeppelin.rest.message.*; import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.server.JsonResponse; import org.apache.zeppelin.service.AuthenticationService; @@ -339,6 +332,31 @@ public class NotebookRestApi extends AbstractRestApi { note -&g
[zeppelin] branch master updated: [ZEPPELIN-5636] Improvement: Automatically clean old version for /interpreter/zeppelin-interpreter-shaded-*.jar
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new ddcbaff [ZEPPELIN-5636] Improvement: Automatically clean old version for /interpreter/zeppelin-interpreter-shaded-*.jar ddcbaff is described below commit ddcbaffdd576a4dfa235541789ca94b84b7b4e1a Author: huage1994 AuthorDate: Mon Jan 17 18:37:32 2022 +0800 [ZEPPELIN-5636] Improvement: Automatically clean old version for /interpreter/zeppelin-interpreter-shaded-*.jar ### What is this PR for? This PR is to automatically clean old version for /interpreter/zeppelin-interpreter-shaded-*.jar. If there are more than one version of /interpreter/zeppelin-interpreter-shaded-*.jar, all version of them would be added into javapath when Zeppelin interpreter starts, which would cause a lot of unexpected problems. And this situation is easy to be ignored when developing. ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? * Open an issue on Jira https://issues.apache.org/jira/projects/ZEPPELIN/issues/ZEPPELIN-5636 ### How should this be tested? CI passed ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: huage1994 Closes #4290 from huage1994/ZEPPELIN-5636 and squashes the following commits: 1f01b8345a [huage1994] [ZEPPELIN-5636] Improvement: Automatically clean old version for /interpreter/zeppelin-interpreter-shaded-*.jar --- zeppelin-interpreter-shaded/pom.xml | 4 1 file changed, 4 insertions(+) diff --git a/zeppelin-interpreter-shaded/pom.xml b/zeppelin-interpreter-shaded/pom.xml index ee53328..52fefb1 100644 --- a/zeppelin-interpreter-shaded/pom.xml +++ b/zeppelin-interpreter-shaded/pom.xml @@ -155,6 +155,10 @@ +ANT TASK - clean files + + + ANT TASK - copying files
[zeppelin] branch branch-0.10 updated: [ZEPPELIN-5619] Fix 'import ipynb/json notebook' button bug
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new 5e75c0b [ZEPPELIN-5619] Fix 'import ipynb/json notebook' button bug 5e75c0b is described below commit 5e75c0b160d43dc5b6effd3ad8d29d30e50c76a8 Author: Heekang Koo AuthorDate: Wed Dec 22 02:02:09 2021 +0900 [ZEPPELIN-5619] Fix 'import ipynb/json notebook' button bug ### What is this PR for? Fixed 'import note - select json/ipynb file' button bug This is my first contribution to open source, so if there is any problem please let me know ### What type of PR is it? Bug Fix ### Todos No ### What is the Jira issue? [[ZEPPELIN-5619]](https://issues.apache.org/jira/browse/ZEPPELIN-5619) ### How should this be tested? 1. Import some local ipynb file(notebook1.ipynb) via ‘Import New Note’ 2. Import another ipynb file(notebook2.ipynb) with the same note name(notebook1). This will make some error message because there exists a zeppelin notebook with the same name(notebook1) in the path 3. Again, import the same ipynb file(notebook2.ipynb) used at step2 without changing note name. Importing notebook process is successfully triggered. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Heekang Koo Closes #4278 from lovtens/notebook-import-fix and squashes the following commits: 32d62c90ae [Heekang Koo] Fix import ipynb/json notebook button (cherry picked from commit de3a88ee1732ece6ca167f3399f759c43c7cb76d) Signed-off-by: Jeff Zhang --- zeppelin-web/src/components/note-import/note-import.controller.js | 4 zeppelin-web/src/components/note-import/note-import.html | 1 + 2 files changed, 5 insertions(+) diff --git a/zeppelin-web/src/components/note-import/note-import.controller.js b/zeppelin-web/src/components/note-import/note-import.controller.js index 45d8279..61e21c8 100644 --- a/zeppelin-web/src/components/note-import/note-import.controller.js +++ b/zeppelin-web/src/components/note-import/note-import.controller.js @@ -43,6 +43,10 @@ function NoteImportCtrl($scope, $timeout, websocketMsgSrv) { angular.element('#noteImportFile').click(); }; + $scope.resetFile = function(element) { +element.value = ''; + }; + $scope.importFile = function(element) { $scope.note.errorText = ''; $scope.note.importFile = element.files[0]; diff --git a/zeppelin-web/src/components/note-import/note-import.html b/zeppelin-web/src/components/note-import/note-import.html index f52ce66..726f449 100644 --- a/zeppelin-web/src/components/note-import/note-import.html +++ b/zeppelin-web/src/components/note-import/note-import.html @@ -47,6 +47,7 @@ limitations under the License.
[zeppelin] branch master updated: [ZEPPELIN-5619] Fix 'import ipynb/json notebook' button bug
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new de3a88e [ZEPPELIN-5619] Fix 'import ipynb/json notebook' button bug de3a88e is described below commit de3a88ee1732ece6ca167f3399f759c43c7cb76d Author: Heekang Koo AuthorDate: Wed Dec 22 02:02:09 2021 +0900 [ZEPPELIN-5619] Fix 'import ipynb/json notebook' button bug ### What is this PR for? Fixed 'import note - select json/ipynb file' button bug This is my first contribution to open source, so if there is any problem please let me know ### What type of PR is it? Bug Fix ### Todos No ### What is the Jira issue? [[ZEPPELIN-5619]](https://issues.apache.org/jira/browse/ZEPPELIN-5619) ### How should this be tested? 1. Import some local ipynb file(notebook1.ipynb) via ‘Import New Note’ 2. Import another ipynb file(notebook2.ipynb) with the same note name(notebook1). This will make some error message because there exists a zeppelin notebook with the same name(notebook1) in the path 3. Again, import the same ipynb file(notebook2.ipynb) used at step2 without changing note name. Importing notebook process is successfully triggered. ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? no * Is there breaking changes for older versions? no * Does this needs documentation? no Author: Heekang Koo Closes #4278 from lovtens/notebook-import-fix and squashes the following commits: 32d62c90ae [Heekang Koo] Fix import ipynb/json notebook button --- zeppelin-web/src/components/note-import/note-import.controller.js | 4 zeppelin-web/src/components/note-import/note-import.html | 1 + 2 files changed, 5 insertions(+) diff --git a/zeppelin-web/src/components/note-import/note-import.controller.js b/zeppelin-web/src/components/note-import/note-import.controller.js index 45d8279..61e21c8 100644 --- a/zeppelin-web/src/components/note-import/note-import.controller.js +++ b/zeppelin-web/src/components/note-import/note-import.controller.js @@ -43,6 +43,10 @@ function NoteImportCtrl($scope, $timeout, websocketMsgSrv) { angular.element('#noteImportFile').click(); }; + $scope.resetFile = function(element) { +element.value = ''; + }; + $scope.importFile = function(element) { $scope.note.errorText = ''; $scope.note.importFile = element.files[0]; diff --git a/zeppelin-web/src/components/note-import/note-import.html b/zeppelin-web/src/components/note-import/note-import.html index f52ce66..726f449 100644 --- a/zeppelin-web/src/components/note-import/note-import.html +++ b/zeppelin-web/src/components/note-import/note-import.html @@ -47,6 +47,7 @@ limitations under the License.
[zeppelin] branch branch-0.10 updated: [ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of detect it at runtime
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new 366b812 [ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of detect it at runtime 366b812 is described below commit 366b812c2840ea4836fd31df95b0436ffa8a52b9 Author: huage1994 AuthorDate: Mon Nov 22 17:46:41 2021 +0800 [ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of detect it at runtime Currently we detect scala version via scala.util.Properties.versionString(); but it depends on the resource file library.version on classpath, sometimes user may package this resource of scala-2.11 into his jar which cause we detect the wrong scala version. [Bug Fix] * [ ] - Task https://issues.apache.org/jira/browse/ZEPPELIN-5580 CI pass * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: huage1994 Closes #4267 from huage1994/ZEPPELIN-5580 and squashes the following commits: 7945faa7e6 [huage1994] [ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of detect it at runtime (cherry picked from commit d5d26583f723d9638d69020aab885afefab82c5c) Signed-off-by: Jeff Zhang --- .../apache/zeppelin/spark/SparkInterpreter.java| 27 -- .../launcher/SparkInterpreterLauncher.java | 9 +++- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 7b1460a..a4c9b76 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -74,6 +74,7 @@ public class SparkInterpreter extends AbstractInterpreter { private Object sparkSession; private SparkVersion sparkVersion; + private String scalaVersion; private boolean enableSupportedVersionCheck; public SparkInterpreter(Properties properties) { @@ -151,7 +152,7 @@ public class SparkInterpreter extends AbstractInterpreter { * @throws Exception */ private AbstractSparkScalaInterpreter loadSparkScalaInterpreter(SparkConf conf) throws Exception { -String scalaVersion = extractScalaVersion(); +scalaVersion = extractScalaVersion(conf); ClassLoader scalaInterpreterClassLoader = Thread.currentThread().getContextClassLoader(); String zeppelinHome = System.getenv("ZEPPELIN_HOME"); @@ -254,14 +255,26 @@ public class SparkInterpreter extends AbstractInterpreter { return sparkVersion; } - private String extractScalaVersion() throws InterpreterException { -String scalaVersionString = scala.util.Properties.versionString(); + private String extractScalaVersion(SparkConf conf) throws InterpreterException { +// Use the scala version if SparkLauncher pass it by name of "zeppelin.spark.scala.version". + +// If not, detect scala version by resource file library.version on classpath. +// Library.version is sometimes inaccurate and it is mainly used for unit test. +String scalaVersionString; +if (conf.contains("zeppelin.spark.scala.version")) { + scalaVersionString = conf.get("zeppelin.spark.scala.version"); +} else { + scalaVersionString = scala.util.Properties.versionString(); +} LOGGER.info("Using Scala: " + scalaVersionString); -if (scalaVersionString.contains("version 2.10")) { + +if (StringUtils.isEmpty(scalaVersionString)) { + throw new InterpreterException("Scala Version is empty"); +} else if (scalaVersionString.contains("2.10")) { return "2.10"; -} else if (scalaVersionString.contains("version 2.11")) { +} else if (scalaVersionString.contains("2.11")) { return "2.11"; -} else if (scalaVersionString.contains("version 2.12")) { +} else if (scalaVersionString.contains("2.12")) { return "2.12"; } else { throw new InterpreterException("Unsupported scala version: " + scalaVersionString); @@ -269,7 +282,7 @@ public class SparkInterpreter extends AbstractInterpreter { } public boolean isScala212() throws InterpreterException { -return extractScalaVersion().equals("2.12"); +return scalaVersion.equals("2.12"); } public boolean isScala210() throws InterpreterException { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.ja
[zeppelin] branch master updated: [ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of detect it at runtime
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new d5d2658 [ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of detect it at runtime d5d2658 is described below commit d5d26583f723d9638d69020aab885afefab82c5c Author: huage1994 AuthorDate: Mon Nov 22 17:46:41 2021 +0800 [ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of detect it at runtime ### What is this PR for? Currently we detect scala version via scala.util.Properties.versionString(); but it depends on the resource file library.version on classpath, sometimes user may package this resource of scala-2.11 into his jar which cause we detect the wrong scala version. ### What type of PR is it? [Bug Fix] ### Todos * [ ] - Task ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-5580 ### How should this be tested? CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: huage1994 Closes #4267 from huage1994/ZEPPELIN-5580 and squashes the following commits: 7945faa7e6 [huage1994] [ZEPPELIN-5580] Pass scala version from SparkInterpreterLauncher instead of detect it at runtime --- .../apache/zeppelin/spark/SparkInterpreter.java| 25 -- .../launcher/SparkInterpreterLauncher.java | 9 +++- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index 16faf99..fef0998 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -74,6 +74,7 @@ public class SparkInterpreter extends AbstractInterpreter { private Object sparkSession; private SparkVersion sparkVersion; + private String scalaVersion; private boolean enableSupportedVersionCheck; public SparkInterpreter(Properties properties) { @@ -149,7 +150,7 @@ public class SparkInterpreter extends AbstractInterpreter { * @throws Exception */ private AbstractSparkScalaInterpreter loadSparkScalaInterpreter(SparkConf conf) throws Exception { -String scalaVersion = extractScalaVersion(); +scalaVersion = extractScalaVersion(conf); ClassLoader scalaInterpreterClassLoader = Thread.currentThread().getContextClassLoader(); String zeppelinHome = System.getenv("ZEPPELIN_HOME"); @@ -252,12 +253,24 @@ public class SparkInterpreter extends AbstractInterpreter { return sparkVersion; } - private String extractScalaVersion() throws InterpreterException { -String scalaVersionString = scala.util.Properties.versionString(); + private String extractScalaVersion(SparkConf conf) throws InterpreterException { +// Use the scala version if SparkLauncher pass it by name of "zeppelin.spark.scala.version". + +// If not, detect scala version by resource file library.version on classpath. +// Library.version is sometimes inaccurate and it is mainly used for unit test. +String scalaVersionString; +if (conf.contains("zeppelin.spark.scala.version")) { + scalaVersionString = conf.get("zeppelin.spark.scala.version"); +} else { + scalaVersionString = scala.util.Properties.versionString(); +} LOGGER.info("Using Scala: " + scalaVersionString); -if (scalaVersionString.contains("version 2.11")) { + +if (StringUtils.isEmpty(scalaVersionString)) { + throw new InterpreterException("Scala Version is empty"); +} else if (scalaVersionString.contains("2.11")) { return "2.11"; -} else if (scalaVersionString.contains("version 2.12")) { +} else if (scalaVersionString.contains("2.12")) { return "2.12"; } else { throw new InterpreterException("Unsupported scala version: " + scalaVersionString); @@ -265,7 +278,7 @@ public class SparkInterpreter extends AbstractInterpreter { } public boolean isScala212() throws InterpreterException { -return extractScalaVersion().equals("2.12"); +return scalaVersion.equals("2.12"); } private List getDependencyFiles() throws InterpreterException { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpret
svn commit: r51027 - /dev/zeppelin/zeppelin-0.10.1-rc1/
Author: zjffdu Date: Mon Nov 22 07:10:32 2021 New Revision: 51027 Log: Upload 0.10.1 rc1 Added: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz (with props) dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz.asc dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz.sha512 dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz (with props) dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz.asc dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz.sha512 dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1.tgz (with props) dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1.tgz.asc dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1.tgz.sha512 Added: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz == Binary file - no diff available. Propchange: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz -- svn:mime-type = application/octet-stream Added: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz.asc == --- dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz.asc (added) +++ dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz.asc Mon Nov 22 07:10:32 2021 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCAAdFiEE9k6XRfAhCMKdxRepCQ++FNmxfR8FAmGbPu4ACgkQCQ++FNmx +fR/pcg//e5UqdL9LYYph+AYsjODN89Au4vayNNV58Y9mePyI8HrlqvOweVXf2s3C +57Tc+NR7U/u2yta2iyyEbMhidZ4TBuf+/GlsPXm7S6KuH0s7u6dy31DyW7Ln+mMb +837A9Yvq8LobQGfyEYo7yYkn9eu9+o/B0fjs0tiwrVt49+J6CJVcgwjPjhZwp9aK +bltAExWJtC8AL2evCwP4Js+utZFrj1s4rByUVROSSLzemjtZ5aRpMh2/vsosk6dp +TId8jlUEtwkVUfIJOaClO13r2qCCJBTbuwkg7okFO7EzTiq3T+Io+JIS9A7w8ku6 +5xgkURF54piCVeHLyT9+vZ702jLSp0Q6gX3Kp23pyIwwukDt+POYlveXjk/DSnSq +ufzGTcJQFdBaCJt/0SIqg/mBilmvbwxKOxHIlGZC/J7xJXCO6FahgvW11dpZnb78 +EwgpkakX4hcR7pBdcQISRFFxiWk41GnNxRVMuv6H7giMVpZpf7O1kfomYHIh6J10 +fu6VMkR4b8HZ+7X6dxeppCJpF/B7nXlpOvINTe4ub/eYhemb6X3PtntU1aT1gSxG +mOzdBFg9ML3Khx4ZfAt4kzY6GUo1SVLSEhZFSTYzZOEbprckDqZuYn18IFcsRMtR +NOAqfaU4Qaw4CjVEQul9XW/fMhX/RKzkPZBSGkc7GIrDclP2GWo= +=9+KE +-END PGP SIGNATURE- Added: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz.sha512 == --- dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz.sha512 (added) +++ dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-all.tgz.sha512 Mon Nov 22 07:10:32 2021 @@ -0,0 +1 @@ +890de6e014da716693fb6e7d76e8c14987d306ff3df02dae5f5e16f566027d2d0c37c3a7dcf913e2948328019d977d1799cc2a2710bd923da809ff16c3dcb9b2 zeppelin-0.10.1-bin-all.tgz Added: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz == Binary file - no diff available. Propchange: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz -- svn:mime-type = application/octet-stream Added: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz.asc == --- dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz.asc (added) +++ dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz.asc Mon Nov 22 07:10:32 2021 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIzBAABCAAdFiEE9k6XRfAhCMKdxRepCQ++FNmxfR8FAmGbOV4ACgkQCQ++FNmx +fR+WahAAiYhLGokGepGlM7nbQXUHXLMVUTd4MD6tgh6ZJHJ4MJ3oWOeVrIOzQKbp +A/3s9pTNE8RSthR1spqOax1jb+r0V5EpAy4ZMBgeQMhKUVTe0Lql2oTqi46NHbiE +EhuoqamAYgzbTxNrEzcflQwNtbz2vtQnhbTxyaA+N1bnduYCblGFCdmoPpyIxbj3 +wwUmDHU0Td5QEjtRIYCGUnTN2QblrLjsdza4yZknfghrF/93clwSn3tSkWtXzc36 +JIIxvae31nJV3ZFni4IH7eIi0CnCCkm/3eN1hkf7HbPlCtM9kFmsTE4axbIKxDf2 +8ZNX0o6P+xVrENHiDAz6yHsKZQdRTLPR3R2Cly34qbHizbIdXC3FsdPwRiWsnUM5 +PvT3Lq64HQF5wXazKB1J6EpIoZJKu2iXfVrp6vcapK1+uCf+xHFOWz3ODHbiZ9z2 +qUqf32q0BrPjAE7H0RvNPDTEAKHbfE5VeWZmIXLCtBv3twanjgjZnr7O8NXrb0MD +SrnZALdfDeqfNA1ZDBRNMLPQrYb7SP8H/d2UuvCc9ngh/7r4gc6i3Aqw24N4bgHG +YCvXeptqxvYZfKbcjPWijzfTsLrY71EDFLkgyzderIrRONd4QI/VHsJU8ewFLxWl +f8HEoTxDItrRNRK5EpVf61XSOm2Z2xwQzqSHUDcsqCkK0zLXCjA= +=LMLs +-END PGP SIGNATURE- Added: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz.sha512 == --- dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz.sha512 (added) +++ dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1-bin-netinst.tgz.sha512 Mon Nov 22 07:10:32 2021 @@ -0,0 +1 @@ +35216f5d81ce74dd7c2dff6c18fd9869c4a1537a62785801b143cef0363a22c58f514c8159bc43b37ecc5f401532e0fddb8f6ffa2f73e3249ad04ed0ee56e4df zeppelin-0.10.1-bin-netinst.tgz Added: dev/zeppelin/zeppelin-0.10.1-rc1/zeppelin-0.10.1
svn commit: r51026 - /dev/zeppelin/zeppelin-0.10.1-rc1/
Author: zjffdu Date: Mon Nov 22 06:55:29 2021 New Revision: 51026 Log: Add folder of 0.10.1 rc1 Added: dev/zeppelin/zeppelin-0.10.1-rc1/
[zeppelin] branch branch-0.10 updated (f82d674 -> 8e70266)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git. from f82d674 [HotFix] Update zeppelin version in Dockerfile add 0a0465c Preparing Apache Zeppelin release 0.10.1 new 8e70266 Preparing development version 0.10.2-SNAPSHOT The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: alluxio/pom.xml | 2 +- angular/pom.xml | 2 +- beam/pom.xml| 2 +- bigquery/pom.xml| 2 +- cassandra/pom.xml | 2 +- docs/_config.yml| 4 ++-- elasticsearch/pom.xml | 2 +- file/pom.xml| 2 +- flink-cmd/pom.xml | 2 +- flink/flink-scala-2.11/pom.xml | 4 ++-- flink/flink-scala-2.12/pom.xml | 4 ++-- flink/flink-scala-parent/pom.xml| 4 ++-- flink/flink-shims/pom.xml | 2 +- flink/flink1.10-shims/pom.xml | 2 +- flink/flink1.11-shims/pom.xml | 2 +- flink/flink1.12-shims/pom.xml | 2 +- flink/flink1.13-shims/pom.xml | 4 ++-- flink/flink1.14-shims/pom.xml | 4 ++-- flink/pom.xml | 2 +- geode/pom.xml | 2 +- groovy/pom.xml | 2 +- hazelcastjet/pom.xml| 2 +- hbase/pom.xml | 2 +- helium-dev/pom.xml | 2 +- ignite/pom.xml | 2 +- influxdb/pom.xml| 2 +- java/pom.xml| 2 +- jdbc/pom.xml| 2 +- kotlin/pom.xml | 2 +- ksql/pom.xml| 2 +- kylin/pom.xml | 2 +- lens/pom.xml| 2 +- livy/pom.xml| 2 +- markdown/pom.xml| 2 +- mongodb/pom.xml | 2 +- neo4j/pom.xml | 2 +- pig/pom.xml | 2 +- pom.xml | 2 +- python/pom.xml | 2 +- rlang/pom.xml | 2 +- sap/pom.xml | 2 +- scalding/pom.xml| 2 +- scio/pom.xml| 2 +- scripts/docker/zeppelin/bin/Dockerfile | 2 +- shell/pom.xml | 2 +- spark-submit/pom.xml| 2 +- spark/interpreter/pom.xml | 2 +- spark/pom.xml | 2 +- spark/scala-2.10/pom.xml| 2 +- spark/scala-2.11/pom.xml| 2 +- spark/scala-2.12/pom.xml| 2 +- spark/spark-dependencies/pom.xml| 2 +- spark/spark-scala-parent/pom.xml| 2 +- spark/spark-shims/pom.xml | 2 +- spark/spark1-shims/pom.xml | 2 +- spark/spark2-shims/pom.xml | 2 +- spark/spark3-shims/pom.xml | 2 +- sparql/pom.xml | 2 +- submarine/pom.xml | 6 +++--- zeppelin-client-examples/pom.xml| 2 +- zeppelin-client/pom.xml | 2 +- zeppelin-common/pom.xml | 2 +- zeppelin-display/pom.xml| 2 +- zeppelin-distribution/pom.xml
[zeppelin] 01/01: Preparing development version 0.10.2-SNAPSHOT
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git commit 8e70266fd14c32e95a333e5183ca150c51b3b165 Author: jeffzhang.zjf AuthorDate: Fri Nov 19 22:38:15 2021 +0800 Preparing development version 0.10.2-SNAPSHOT --- alluxio/pom.xml | 2 +- angular/pom.xml | 2 +- beam/pom.xml| 2 +- bigquery/pom.xml| 2 +- cassandra/pom.xml | 2 +- elasticsearch/pom.xml | 2 +- file/pom.xml| 2 +- flink-cmd/pom.xml | 2 +- flink/flink-scala-2.11/pom.xml | 4 ++-- flink/flink-scala-2.12/pom.xml | 4 ++-- flink/flink-scala-parent/pom.xml| 4 ++-- flink/flink-shims/pom.xml | 2 +- flink/flink1.10-shims/pom.xml | 2 +- flink/flink1.11-shims/pom.xml | 2 +- flink/flink1.12-shims/pom.xml | 2 +- flink/flink1.13-shims/pom.xml | 4 ++-- flink/flink1.14-shims/pom.xml | 4 ++-- flink/pom.xml | 2 +- geode/pom.xml | 2 +- groovy/pom.xml | 2 +- hazelcastjet/pom.xml| 2 +- hbase/pom.xml | 2 +- helium-dev/pom.xml | 2 +- ignite/pom.xml | 2 +- influxdb/pom.xml| 2 +- java/pom.xml| 2 +- jdbc/pom.xml| 2 +- kotlin/pom.xml | 2 +- ksql/pom.xml| 2 +- kylin/pom.xml | 2 +- lens/pom.xml| 2 +- livy/pom.xml| 2 +- markdown/pom.xml| 2 +- mongodb/pom.xml | 2 +- neo4j/pom.xml | 2 +- pig/pom.xml | 2 +- pom.xml | 2 +- python/pom.xml | 2 +- rlang/pom.xml | 2 +- sap/pom.xml | 2 +- scalding/pom.xml| 2 +- scio/pom.xml| 2 +- scripts/docker/zeppelin/bin/Dockerfile | 2 +- shell/pom.xml | 2 +- spark-submit/pom.xml| 2 +- spark/interpreter/pom.xml | 2 +- spark/pom.xml | 2 +- spark/scala-2.10/pom.xml| 2 +- spark/scala-2.11/pom.xml| 2 +- spark/scala-2.12/pom.xml| 2 +- spark/spark-dependencies/pom.xml| 2 +- spark/spark-scala-parent/pom.xml| 2 +- spark/spark-shims/pom.xml | 2 +- spark/spark1-shims/pom.xml | 2 +- spark/spark2-shims/pom.xml | 2 +- spark/spark3-shims/pom.xml | 2 +- sparql/pom.xml | 2 +- submarine/pom.xml | 6 +++--- zeppelin-client-examples/pom.xml| 2 +- zeppelin-client/pom.xml | 2 +- zeppelin-common/pom.xml | 2 +- zeppelin-display/pom.xml| 2 +- zeppelin-distribution/pom.xml | 2 +- zeppelin-examples/pom.xml | 2 +- zeppelin-examples/zeppelin-example-clock/pom.xml| 4 ++-- zeppelin-examples/zeppelin-example-horizontalbar/pom.xml| 4 ++-- zeppelin-examples/zeppelin-example-spell-echo/pom.xml | 4 ++-- zeppelin-examples/zeppelin-example-spell-flowchart/pom.xml | 4
[zeppelin] 01/01: Preparing Apache Zeppelin release 0.10.1
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to tag v0.10.1-rc1 in repository https://gitbox.apache.org/repos/asf/zeppelin.git commit 0a0465c1930cf6775149b5609d5da3df47383896 Author: jeffzhang.zjf AuthorDate: Fri Nov 19 22:38:04 2021 +0800 Preparing Apache Zeppelin release 0.10.1 --- alluxio/pom.xml | 2 +- angular/pom.xml | 2 +- beam/pom.xml| 2 +- bigquery/pom.xml| 2 +- cassandra/pom.xml | 2 +- docs/_config.yml| 4 ++-- elasticsearch/pom.xml | 2 +- file/pom.xml| 2 +- flink-cmd/pom.xml | 2 +- flink/flink-scala-2.11/pom.xml | 4 ++-- flink/flink-scala-2.12/pom.xml | 4 ++-- flink/flink-scala-parent/pom.xml| 4 ++-- flink/flink-shims/pom.xml | 2 +- flink/flink1.10-shims/pom.xml | 2 +- flink/flink1.11-shims/pom.xml | 2 +- flink/flink1.12-shims/pom.xml | 2 +- flink/flink1.13-shims/pom.xml | 4 ++-- flink/flink1.14-shims/pom.xml | 4 ++-- flink/pom.xml | 2 +- geode/pom.xml | 2 +- groovy/pom.xml | 2 +- hazelcastjet/pom.xml| 2 +- hbase/pom.xml | 2 +- helium-dev/pom.xml | 2 +- ignite/pom.xml | 2 +- influxdb/pom.xml| 2 +- java/pom.xml| 2 +- jdbc/pom.xml| 2 +- kotlin/pom.xml | 2 +- ksql/pom.xml| 2 +- kylin/pom.xml | 2 +- lens/pom.xml| 2 +- livy/pom.xml| 2 +- markdown/pom.xml| 2 +- mongodb/pom.xml | 2 +- neo4j/pom.xml | 2 +- pig/pom.xml | 2 +- pom.xml | 2 +- python/pom.xml | 2 +- rlang/pom.xml | 2 +- sap/pom.xml | 2 +- scalding/pom.xml| 2 +- scio/pom.xml| 2 +- shell/pom.xml | 2 +- spark-submit/pom.xml| 2 +- spark/interpreter/pom.xml | 2 +- spark/pom.xml | 2 +- spark/scala-2.10/pom.xml| 2 +- spark/scala-2.11/pom.xml| 2 +- spark/scala-2.12/pom.xml| 2 +- spark/spark-dependencies/pom.xml| 2 +- spark/spark-scala-parent/pom.xml| 2 +- spark/spark-shims/pom.xml | 2 +- spark/spark1-shims/pom.xml | 2 +- spark/spark2-shims/pom.xml | 2 +- spark/spark3-shims/pom.xml | 2 +- sparql/pom.xml | 2 +- submarine/pom.xml | 6 +++--- zeppelin-client-examples/pom.xml| 2 +- zeppelin-client/pom.xml | 2 +- zeppelin-common/pom.xml | 2 +- zeppelin-display/pom.xml| 2 +- zeppelin-distribution/pom.xml | 2 +- zeppelin-examples/pom.xml | 2 +- zeppelin-examples/zeppelin-example-clock/pom.xml| 4 ++-- zeppelin-examples/zeppelin-example-horizontalbar/pom.xml| 4 ++-- zeppelin-examples/zeppelin-example-spell-echo/pom.xml | 4 ++-- zeppelin-examples/zeppelin-example-spell-flowchart/pom.xml | 4 ++-- zeppelin
[zeppelin] tag v0.10.1-rc1 created (now 0a0465c)
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a change to tag v0.10.1-rc1 in repository https://gitbox.apache.org/repos/asf/zeppelin.git. at 0a0465c (commit) This tag includes the following new commits: new 0a0465c Preparing Apache Zeppelin release 0.10.1 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[zeppelin] branch branch-0.10 updated: [HotFix] Update zeppelin version in Dockerfile
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new f82d674 [HotFix] Update zeppelin version in Dockerfile f82d674 is described below commit f82d674b50901f5faba03d13fd809a36883c8ea0 Author: Jeff Zhang AuthorDate: Fri Nov 19 22:34:30 2021 +0800 [HotFix] Update zeppelin version in Dockerfile --- k8s/zeppelin-server.yaml | 4 ++-- scripts/docker/zeppelin-interpreter/Dockerfile | 2 +- scripts/docker/zeppelin-server/Dockerfile | 2 +- scripts/docker/zeppelin/bin/Dockerfile | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/k8s/zeppelin-server.yaml b/k8s/zeppelin-server.yaml index 2988b6f..ff7b472 100644 --- a/k8s/zeppelin-server.yaml +++ b/k8s/zeppelin-server.yaml @@ -29,7 +29,7 @@ data: # If you have your ingress controller configured to connect to `zeppelin-server` service and have a domain name for it (with wildcard subdomain point the same address), you can replace serviceDomain field with your own domain. SERVICE_DOMAIN: local.zeppelin-project.org:8080 ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE: spark:2.4.5 - ZEPPELIN_K8S_CONTAINER_IMAGE: apache/zeppelin-interpreter:0.10.0 + ZEPPELIN_K8S_CONTAINER_IMAGE: apache/zeppelin-interpreter:0.10.1 ZEPPELIN_HOME: /opt/zeppelin ZEPPELIN_SERVER_RPC_PORTRANGE: 12320:12320 # default value of 'master' property for spark interpreter. @@ -115,7 +115,7 @@ spec: path: nginx.conf containers: - name: zeppelin-server -image: apache/zeppelin-server:0.10.0 +image: apache/zeppelin-server:0.10.1 command: ["sh", "-c", "$(ZEPPELIN_HOME)/bin/zeppelin.sh"] lifecycle: preStop: diff --git a/scripts/docker/zeppelin-interpreter/Dockerfile b/scripts/docker/zeppelin-interpreter/Dockerfile index dc78370..29cc97c 100644 --- a/scripts/docker/zeppelin-interpreter/Dockerfile +++ b/scripts/docker/zeppelin-interpreter/Dockerfile @@ -19,7 +19,7 @@ FROM ubuntu:20.04 LABEL maintainer="Apache Software Foundation " -ARG version="0.10.0" +ARG version="0.10.1" ENV VERSION="${version}" \ ZEPPELIN_HOME="/opt/zeppelin" diff --git a/scripts/docker/zeppelin-server/Dockerfile b/scripts/docker/zeppelin-server/Dockerfile index 1cd18c5..2ee91ea 100644 --- a/scripts/docker/zeppelin-server/Dockerfile +++ b/scripts/docker/zeppelin-server/Dockerfile @@ -35,7 +35,7 @@ RUN set -ex && \ apt-get autoclean && \ apt-get clean -ARG version="0.10.0" +ARG version="0.10.1" ENV LANG=en_US.UTF-8 \ LC_ALL=en_US.UTF-8 \ diff --git a/scripts/docker/zeppelin/bin/Dockerfile b/scripts/docker/zeppelin/bin/Dockerfile index dab26bd..8deb91c 100644 --- a/scripts/docker/zeppelin/bin/Dockerfile +++ b/scripts/docker/zeppelin/bin/Dockerfile @@ -17,7 +17,7 @@ FROM ubuntu:20.04 LABEL maintainer="Apache Software Foundation " -ENV Z_VERSION="0.10.0" +ENV Z_VERSION="0.10.1" ENV LOG_TAG="[ZEPPELIN_${Z_VERSION}]:" \ ZEPPELIN_HOME="/opt/zeppelin" \
[zeppelin] branch master updated: [HotFix] Flink UT is not enabled
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 2c98720 [HotFix] Flink UT is not enabled 2c98720 is described below commit 2c98720e1bfa304bca4138187b885cfccb64718f Author: Jeff Zhang AuthorDate: Thu Oct 28 18:22:05 2021 +0800 [HotFix] Flink UT is not enabled --- .github/workflows/core.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 1503c66..efe147f 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -181,7 +181,7 @@ jobs: ./mvnw install -DskipTests -DskipRat -am -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -B ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests -B - name: run tests -run: ./mvnw test -DskipRat -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -DfailIfNoTests=false -B -Dtest=ZeppelinFlinkClusterTest${{ matrix.flink }} +run: ./mvnw test -DskipRat -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -DfailIfNoTests=false -B -Dtest=org.apache.zeppelin.flink.*,FlinkIntegrationTest${{ matrix.flink }} run-spark-intergration-test: runs-on: ubuntu-20.04
[zeppelin] branch branch-0.10 updated: [ZEPPELIN-5565] Support Spark 3.2
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new 2128a8d [ZEPPELIN-5565] Support Spark 3.2 2128a8d is described below commit 2128a8d42c21c2c36b1253e552bce1ed290fc43f Author: Jeff Zhang AuthorDate: Mon Oct 25 15:53:13 2021 +0800 [ZEPPELIN-5565] Support Spark 3.2 Simple PR to support Spark 3.2. No many code change in Spark module, because Spark 3.2 is compatible with Spark 3.1 which we have already supported. Only netty is excluded in pom.xml because it conflicts with the netty that Spark 3.2 use. This only affect the Spark unit test. Integration test still pass even without these netty change, because unit test use different classpath. [ Improvement ] * [ ] - Task * https://issues.apache.org/jira/browse/ZEPPELIN-5565 * CI pass * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4257 from zjffdu/ZEPPELIN-5565 and squashes the following commits: 9b5b6c42d7 [Jeff Zhang] Update doc 7b8924a850 [Jeff Zhang] Update scala version 1b20f934b1 [Jeff Zhang] [ZEPPELIN-5565] Support Spark 3.2 (cherry picked from commit 2f55fe8ed277b28d71f858633f9c9d76fd18f0c3) Signed-off-by: Jeff Zhang --- .github/workflows/core.yml | 43 +++- .github/workflows/frontend.yml | 2 +- docs/setup/basics/how_to_build.md | 4 +- spark/interpreter/pom.xml | 12 + .../org/apache/zeppelin/spark/SparkShimsTest.java | 6 +-- spark/pom.xml | 14 +- spark/scala-2.12/pom.xml | 2 +- .../org/apache/zeppelin/spark/SparkVersion.java| 8 ++- .../integration/SparkIntegrationTest32.java| 58 ++ .../integration/ZeppelinSparkClusterTest32.java| 40 +++ 10 files changed, 174 insertions(+), 15 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index e33fe18..29c2a0b 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -217,7 +217,7 @@ jobs: mvn install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B mvn clean package -pl zeppelin-plugins -amd -DskipTests -B - name: run tests -run: mvn test -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B -Dtest=ZeppelinSparkClusterTest24,SparkSubmitIntegrationTest,SparkIntegrationTest24,ZeppelinSparkClusterTest23,SparkIntegrationTest23,ZeppelinSparkClusterTest22,SparkIntegrationTest22,ZeppelinSparkClusterTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest30,SparkIntegrationTest31 -DfailIfNoTests=false +run: mvn test -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B -Dtest=ZeppelinSparkClusterTest24,SparkSubmitIntegrationTest,SparkIntegrationTest24,ZeppelinSparkClusterTest23,SparkIntegrationTest23,ZeppelinSparkClusterTest22,SparkIntegrationTest22,ZeppelinSparkClusterTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest30,SparkIntegrationTest31,ZeppelinSparkClusterTest32,SparkIntegrationTest3 [...] jdbcIntegrationTest-and-unit-test-of-Spark-2-4-with-Scala-2-11: runs-on: ubuntu-20.04 @@ -377,6 +377,47 @@ jobs: run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B - name: run tests with ${{ matrix.python }} run: mvn test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false + + spark-3-2-and-scala-2-12-and-other-interpreter: +runs-on: ubuntu-20.04 +strategy: + fail-fast: false + matrix: +python: [ 3.7, 3.8 ] +steps: + - name: Checkout +uses: actions/checkout@v2 + - name: Tune Runner VM +uses: ./.github/actions/tune-runner-vm + - name: Set up JDK 8 +uses: actions/setup-java@v2 +with: + distribution: 'adopt' + java-version: 8 + - name: Cache local Maven repository +uses: actions/cache@v2 +with: + path: | +~/.m2/repository +!~/.m2/repository/org/apache/zeppelin/ + key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} + restore-keys
[zeppelin] branch master updated: [ZEPPELIN-5565] Support Spark 3.2
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 2f55fe8 [ZEPPELIN-5565] Support Spark 3.2 2f55fe8 is described below commit 2f55fe8ed277b28d71f858633f9c9d76fd18f0c3 Author: Jeff Zhang AuthorDate: Mon Oct 25 15:53:13 2021 +0800 [ZEPPELIN-5565] Support Spark 3.2 ### What is this PR for? Simple PR to support Spark 3.2. No many code change in Spark module, because Spark 3.2 is compatible with Spark 3.1 which we have already supported. Only netty is excluded in pom.xml because it conflicts with the netty that Spark 3.2 use. This only affect the Spark unit test. Integration test still pass even without these netty change, because unit test use different classpath. ### What type of PR is it? [ Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5565 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4257 from zjffdu/ZEPPELIN-5565 and squashes the following commits: 9b5b6c42d7 [Jeff Zhang] Update doc 7b8924a850 [Jeff Zhang] Update scala version 1b20f934b1 [Jeff Zhang] [ZEPPELIN-5565] Support Spark 3.2 --- .github/workflows/core.yml | 43 +++- .github/workflows/frontend.yml | 8 +-- docs/setup/basics/how_to_build.md | 3 +- spark/interpreter/pom.xml | 12 + .../org/apache/zeppelin/spark/SparkShimsTest.java | 6 +-- spark/pom.xml | 14 +- spark/scala-2.12/pom.xml | 2 +- .../org/apache/zeppelin/spark/SparkVersion.java| 8 ++- .../integration/SparkIntegrationTest32.java| 58 ++ .../integration/ZeppelinSparkClusterTest32.java| 40 +++ 10 files changed, 176 insertions(+), 18 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 866ed31..1503c66 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -219,7 +219,7 @@ jobs: ./mvnw install -DskipTests -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests -B - name: run tests -run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B -Dtest=ZeppelinSparkClusterTest24,SparkSubmitIntegrationTest,SparkIntegrationTest24,ZeppelinSparkClusterTest23,SparkIntegrationTest23,ZeppelinSparkClusterTest22,SparkIntegrationTest22,ZeppelinSparkClusterTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest30,SparkIntegrationTest31 -DfailIfNoTests=false +run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration,zeppelin-web,spark-submit,spark/spark-dependencies,markdown -am -Phadoop2 -Pintegration -B -Dtest=SparkSubmitIntegrationTest,ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest30,SparkIntegrationTest30,ZeppelinSparkClusterTest31,SparkIntegrationTest31,ZeppelinSparkClusterTest32,SparkIntegrationTest32 -DfailIfNoTests=false jdbcIntegrationTest-and-unit-test-of-Spark-2-4-with-Scala-2-11: runs-on: ubuntu-20.04 @@ -379,6 +379,47 @@ jobs: run: ./mvnw install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B - name: run tests with ${{ matrix.python }} run: ./mvnw test -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B -Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false + + spark-3-2-and-scala-2-12-and-other-interpreter: +runs-on: ubuntu-20.04 +strategy: + fail-fast: false + matrix: +python: [ 3.7, 3.8 ] +steps: + - name: Checkout +uses: actions/checkout@v2 + - name: Tune Runner VM +uses: ./.github/actions/tune-runner-vm + - name: Set up JDK 8 +uses: actions/setup-java@v2 +with: + distribution: 'adopt' + java-version: 8 + - name: Cache local Maven repository +uses: actions/cache@v2 +with: + path: | +~/.m2/repository +!~/.m2/repository/org/apache/zeppelin/ + key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml
[zeppelin] branch master updated: [ZEPPELIN-5551] add support for trino driver for recent versions
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new cd2b3f4 [ZEPPELIN-5551] add support for trino driver for recent versions cd2b3f4 is described below commit cd2b3f4c95add218cad1c0572f39ef0610841514 Author: jaeho.yoo AuthorDate: Tue Oct 12 20:40:38 2021 +0900 [ZEPPELIN-5551] add support for trino driver for recent versions ### What is this PR for? - Prestosql has changed its name to Trino in 2020.12 and Trino no longer supports the name `io.prestosql.jdbc.PrestoDriver` and has been removed in v358. - Adds support for trino drivers for v358 and above - Users can now use trino named driver for driver name. ### What type of PR is it? - Bug Fix ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-5551 ### How should this be tested? * Test if JDBC Interpreter works for trino/presto ### Screenshots (if appropriate) ![image](https://user-images.githubusercontent.com/25147023/137051013-c3a65d30-9b52-4212-8c13-b7b54a7cbe3a.png) ![image](https://user-images.githubusercontent.com/25147023/137051018-6f2c656c-4b16-410b-963c-d6ec292e01b0.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: jaeho.yoo Closes #4253 from Chaho12/feature/jaeho.yoo/support_trino_jdbc and squashes the following commits: 8da748ec06 [jaeho.yoo] add support for trino driver for recent versions (cherry picked from commit 0c6f6fc17ac2cdbf10f75f52fbb1772bc289a14d) Signed-off-by: Jeff Zhang --- jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index 3af326d..58068e2 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -473,10 +473,14 @@ public class JDBCInterpreter extends KerberosInterpreter { LOGGER.info("Creating connection pool for url: {}, user: {}, dbPrefix: {}, properties: {}", url, user, dbPrefix, properties); +/* Remove properties that is not valid properties for presto/trino by checking driver key. + * - Presto: com.facebook.presto.jdbc.PrestoDriver + * - Trino(ex. PrestoSQL): io.trino.jdbc.TrinoDriver / io.prestosql.jdbc.PrestoDriver + */ String driverClass = properties.getProperty(DRIVER_KEY); if (driverClass != null && (driverClass.equals("com.facebook.presto.jdbc.PrestoDriver") -|| driverClass.equals("io.prestosql.jdbc.PrestoDriver"))) { - // Only add valid properties otherwise presto won't work. +|| driverClass.equals("io.prestosql.jdbc.PrestoDriver") +|| driverClass.equals("io.trino.jdbc.TrinoDriver"))) { for (String key : properties.stringPropertyNames()) { if (!PRESTO_PROPERTIES.contains(key)) { properties.remove(key);
[zeppelin] branch branch-0.10 updated: [ZEPPELIN-5551] add support for trino driver for recent versions
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new 0c6f6fc [ZEPPELIN-5551] add support for trino driver for recent versions 0c6f6fc is described below commit 0c6f6fc17ac2cdbf10f75f52fbb1772bc289a14d Author: jaeho.yoo AuthorDate: Tue Oct 12 20:40:38 2021 +0900 [ZEPPELIN-5551] add support for trino driver for recent versions ### What is this PR for? - Prestosql has changed its name to Trino in 2020.12 and Trino no longer supports the name `io.prestosql.jdbc.PrestoDriver` and has been removed in v358. - Adds support for trino drivers for v358 and above - Users can now use trino named driver for driver name. ### What type of PR is it? - Bug Fix ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-5551 ### How should this be tested? * Test if JDBC Interpreter works for trino/presto ### Screenshots (if appropriate) ![image](https://user-images.githubusercontent.com/25147023/137051013-c3a65d30-9b52-4212-8c13-b7b54a7cbe3a.png) ![image](https://user-images.githubusercontent.com/25147023/137051018-6f2c656c-4b16-410b-963c-d6ec292e01b0.png) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: jaeho.yoo Closes #4253 from Chaho12/feature/jaeho.yoo/support_trino_jdbc and squashes the following commits: 8da748ec06 [jaeho.yoo] add support for trino driver for recent versions --- jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java | 8 ++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java index 3af326d..58068e2 100644 --- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java +++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java @@ -473,10 +473,14 @@ public class JDBCInterpreter extends KerberosInterpreter { LOGGER.info("Creating connection pool for url: {}, user: {}, dbPrefix: {}, properties: {}", url, user, dbPrefix, properties); +/* Remove properties that is not valid properties for presto/trino by checking driver key. + * - Presto: com.facebook.presto.jdbc.PrestoDriver + * - Trino(ex. PrestoSQL): io.trino.jdbc.TrinoDriver / io.prestosql.jdbc.PrestoDriver + */ String driverClass = properties.getProperty(DRIVER_KEY); if (driverClass != null && (driverClass.equals("com.facebook.presto.jdbc.PrestoDriver") -|| driverClass.equals("io.prestosql.jdbc.PrestoDriver"))) { - // Only add valid properties otherwise presto won't work. +|| driverClass.equals("io.prestosql.jdbc.PrestoDriver") +|| driverClass.equals("io.trino.jdbc.TrinoDriver"))) { for (String key : properties.stringPropertyNames()) { if (!PRESTO_PROPERTIES.contains(key)) { properties.remove(key);
[zeppelin] branch master updated: [ZEPPELIN-5553] Remove support of flink 1.10 & 1.11
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 7e6c123 [ZEPPELIN-5553] Remove support of flink 1.10 & 1.11 7e6c123 is described below commit 7e6c1236ff84acd636325763eb86017e8a412d9c Author: Jeff Zhang AuthorDate: Mon Oct 18 11:15:09 2021 +0800 [ZEPPELIN-5553] Remove support of flink 1.10 & 1.11 ### What is this PR for? * Remove module `flink1.10-shims` & `flink1.11-shims` * Related tests are also removed ### What type of PR is it? [ Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5553 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4251 from zjffdu/ZEPPELIN-5553 and squashes the following commits: 1160dc675f [Jeff Zhang] update 929df448f9 [Jeff Zhang] [ZEPPELIN-5553] Remove support of flink 1.10 & 1.11 --- .github/workflows/core.yml | 6 +- flink/README.md| 3 +- flink/flink-scala-parent/pom.xml | 92 +--- .../apache/zeppelin/flink/FlinkSqlInterpreter.java | 12 +- .../apache/zeppelin/flink/IPyFlinkInterpreter.java | 4 - .../java/org/apache/zeppelin/flink/JobManager.java | 1 + .../apache/zeppelin/flink/PyFlinkInterpreter.java | 4 - .../org/apache/zeppelin/flink/TableEnvFactory.java | 58 +-- .../src/main/resources/python/zeppelin_ipyflink.py | 9 +- .../src/main/resources/python/zeppelin_pyflink.py | 9 +- .../zeppelin/flink/FlinkScalaInterpreter.scala | 16 +- .../zeppelin/flink/internal/FlinkShell.scala | 3 - .../flink/FlinkBatchSqlInterpreterTest.java| 4 - .../apache/zeppelin/flink/SqlInterpreterTest.java | 212 - .../java/org/apache/zeppelin/flink/FlinkShims.java | 8 +- .../org/apache/zeppelin/flink/FlinkVersion.java| 4 - flink/flink1.10-shims/pom.xml | 217 - .../org/apache/zeppelin/flink/Flink110Shims.java | 389 --- .../flink/shims110/CollectStreamTableSink.java | 102 .../flink/shims110/Flink110ScalaShims.scala| 37 -- flink/flink1.11-shims/pom.xml | 211 - .../org/apache/zeppelin/flink/Flink111Shims.java | 526 - .../flink/shims111/CollectStreamTableSink.java | 97 .../flink/shims111/Flink111ScalaShims.scala| 36 -- flink/pom.xml | 4 - .../integration/FlinkIntegrationTest110.java | 40 -- .../integration/FlinkIntegrationTest111.java | 40 -- .../integration/ZSessionIntegrationTest.java | 2 +- .../integration/ZeppelinFlinkClusterTest.java | 15 +- .../integration/ZeppelinFlinkClusterTest112.java | 8 +- ...st111.java => ZeppelinFlinkClusterTest113.java} | 9 +- ...st110.java => ZeppelinFlinkClusterTest114.java} | 9 +- .../src/test/resources/init_stream.scala | 2 +- .../interpreter/RemoteInterpreterEventServer.java | 1 + 34 files changed, 140 insertions(+), 2050 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 1609ad1..866ed31 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -143,12 +143,13 @@ jobs: ./mvnw package -DskipRat -pl zeppelin-plugins -amd -DskipTests -B - name: run tests run: ./mvnw test -DskipRat -pl zeppelin-interpreter-integration -Pintegration -DfailIfNoTests=false -Dtest=ZeppelinClientIntegrationTest,ZeppelinClientWithAuthIntegrationTest,ZSessionIntegrationTest + test-flink-and-flink-integration-test: runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: -flink: [110, 111, 112, 113, 114] +flink: [112, 113, 114] steps: - name: Checkout uses: actions/checkout@v2 @@ -180,7 +181,8 @@ jobs: ./mvnw install -DskipTests -DskipRat -am -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -B ./mvnw clean package -pl zeppelin-plugins -amd -DskipTests -B - name: run tests -run: ./mvnw test -DskipRat -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -DfailIfNoTests=false -B -Dtest=org.apache.zeppelin.flink.*,FlinkIntegrationTest${{ matrix.flink }},ZeppelinFlinkClusterTest${{ matrix.flink }} +run: ./mvnw test -DskipRat -pl flink/flink-
[zeppelin] branch master updated: [ZEPPELIN-5560] spark yarn app end with failed status in yarn-cluster mode
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new ddff5f8 [ZEPPELIN-5560] spark yarn app end with failed status in yarn-cluster mode ddff5f8 is described below commit ddff5f8c2ef4758928d2f0eb494b474c43de1a44 Author: Jeff Zhang AuthorDate: Wed Oct 13 14:37:57 2021 +0800 [ZEPPELIN-5560] spark yarn app end with failed status in yarn-cluster mode ### What is this PR for? The root cause is that `RemoteInterpreterServer` would call System.exit to forceShutdown spark driver in yarn-cluster mode. This PR would disable forceShutdown in spark's yarn-cluster mode. ### What type of PR is it? [ Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5560 ### How should this be tested? * CI ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4255 from zjffdu/ZEPPELIN-5560 and squashes the following commits: 5b4aceeb95 [Jeff Zhang] [ZEPPELIN-5560] spark yarn app end with failed status in yarn-cluster mode --- zeppelin-interpreter-integration/README.md | 6 -- .../zeppelin/integration/SparkIntegrationTest.java | 17 - .../interpreter/launcher/SparkInterpreterLauncher.java | 11 ++- .../launcher/SparkInterpreterLauncherTest.java | 2 +- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/zeppelin-interpreter-integration/README.md b/zeppelin-interpreter-integration/README.md index bf235ff..6f2cec9 100644 --- a/zeppelin-interpreter-integration/README.md +++ b/zeppelin-interpreter-integration/README.md @@ -1,4 +1,6 @@ ## How to run Zeppelin integration tests -If you have hadoop installed on your machine, please make sure to unset hadoop related enviromnets: -* HADOOP_CONF_DIR \ No newline at end of file +If you have hadoop installed on your machine, please make sure to unset hadoop related environments: +* HADOOP_CONF_DIR + +If you want to run integration tests in IDE, please set `ZEPPELIN_HOME` manually. diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java index 1dec6ee..2c4c278 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java @@ -20,6 +20,8 @@ package org.apache.zeppelin.integration; import org.apache.commons.io.IOUtils; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.exceptions.YarnException; import org.apache.maven.model.Model; @@ -46,6 +48,8 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; import java.util.EnumSet; +import java.util.List; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -249,6 +253,7 @@ public abstract class SparkIntegrationTest { // parameters with whitespace sparkInterpreterSetting.setProperty("spark.app.name", "hello spark"); +String yarnAppId = null; try { setUpSparkInterpreterSetting(sparkInterpreterSetting); testInterpreterBasics(); @@ -258,10 +263,20 @@ public abstract class SparkIntegrationTest { GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request); assertEquals(1, response.getApplicationList().size()); assertEquals("hello spark", response.getApplicationList().get(0).getName()); - + yarnAppId = response.getApplicationList().get(0).getApplicationId().toString(); } finally { interpreterSettingManager.close(); waitForYarnAppCompleted(30 * 1000); + + if (yarnAppId != null) { +// ensure yarn app is finished with SUCCEEDED status. +final String finalYarnAppId = yarnAppId; +GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.FINISHED)); +GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResour
[zeppelin] branch master updated: [ZEPPEIN-5555] Remove scala-2.10 & spark 1.x support in spark interpreter
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new a0da85d [ZEPPEIN-] Remove scala-2.10 & spark 1.x support in spark interpreter a0da85d is described below commit a0da85dd48f9000bf510167cc72a558008bf651d Author: Jeff Zhang AuthorDate: Thu Oct 14 20:16:35 2021 +0800 [ZEPPEIN-] Remove scala-2.10 & spark 1.x support in spark interpreter ### What is this PR for? This PR remove the support of scala-2.10 & spark 1.x in spark interpreter. ### What type of PR is it? [ Improvement ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN- ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4250 from zjffdu/ZEPPELIN- and squashes the following commits: 9577e21e72 [Jeff Zhang] update doc 0329a40e07 [Jeff Zhang] [ZEPPEIN-] Remove scala-2.10 & spark 1.x support in spark interpreter --- docs/setup/basics/how_to_build.md | 14 +- spark/interpreter/pom.xml | 6 - .../apache/zeppelin/spark/IPySparkInterpreter.java | 4 - .../apache/zeppelin/spark/PySparkInterpreter.java | 4 - .../apache/zeppelin/spark/SparkInterpreter.java| 10 +- .../apache/zeppelin/spark/SparkRInterpreter.java | 19 +-- .../src/main/resources/python/zeppelin_ipyspark.py | 9 +- .../src/main/resources/python/zeppelin_pyspark.py | 9 +- .../zeppelin/spark/IPySparkInterpreterTest.java| 73 --- .../zeppelin/spark/SparkInterpreterTest.java | 9 +- .../zeppelin/spark/SparkRInterpreterTest.java | 70 +- .../zeppelin/spark/SparkShinyInterpreterTest.java | 11 +- .../zeppelin/spark/SparkSqlInterpreterTest.java| 14 +- spark/pom.xml | 50 spark/scala-2.10/pom.xml | 58 - spark/scala-2.10/spark-scala-parent| 1 - .../zeppelin/spark/SparkScala210Interpreter.scala | 119 - .../zeppelin/spark/SparkScala211Interpreter.scala | 1 - spark/spark-dependencies/pom.xml | 7 - .../zeppelin/spark/BaseSparkScalaInterpreter.scala | 46 +-- .../java/org/apache/zeppelin/spark/SparkShims.java | 3 - .../org/apache/zeppelin/spark/SparkVersion.java| 4 - spark/spark1-shims/pom.xml | 86 - .../org/apache/zeppelin/spark/Spark1Shims.java | 141 - 24 files changed, 76 insertions(+), 692 deletions(-) diff --git a/docs/setup/basics/how_to_build.md b/docs/setup/basics/how_to_build.md index 5939ca4..4b1b9a3 100644 --- a/docs/setup/basics/how_to_build.md +++ b/docs/setup/basics/how_to_build.md @@ -106,13 +106,9 @@ Set spark major version Available profiles are ``` +-Pspark-3.1 -Pspark-3.0 -Pspark-2.4 --Pspark-2.3 --Pspark-2.2 --Pspark-2.1 --Pspark-2.0 --Pspark-1.6 ``` minor version can be adjusted by `-Dspark.version=x.x.x` @@ -125,13 +121,12 @@ Actually Zeppelin supports all the versions of scala (2.10, 2.11, 2.12) in Spark Available profiles are ``` --Pspark-scala-2.10 -Pspark-scala-2.11 -Pspark-scala-2.12 ``` If you want to use Spark 3.x in the embedded mode, then you have to specify both profile `spark-3.0` and `spark-scala-2.12`, -because Spark 3.x doesn't support scala 2.10 and 2.11. +because Spark 3.x doesn't support scala 2.11. Build hadoop with Zeppelin (`-Phadoop[version]`) @@ -169,11 +164,6 @@ Here are some examples with several options: # build with spark-2.4, spark-scala-2.11 ./mvnw clean package -Pspark-2.4 -Pspark-scala-2.11 -DskipTests -# build with spark-1.6, spark-scala-2.10 -./mvnw clean package -Pspark-1.6 -Pspark-scala-2.10 -DskipTests - -# build with CDH -./mvnw clean package -Pspark-1.6 -Pspark-scala-2.10 -Dhadoop.version=2.6.0-cdh5.5.0 -Pvendor-repo -DskipTests ``` Ignite Interpreter diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml index 2e9a33f..42ab2cc 100644 --- a/spark/interpreter/pom.xml +++ b/spark/interpreter/pom.xml @@ -82,12 +82,6 @@ org.apache.zeppelin - spark1-shims - ${project.version} - - - - org.apache.zeppelin spark2-shims ${project.version} diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java index 0e3729f..ab6b3db 100644 --- a/spark/interpreter/src/main/java/org/apach
[zeppelin] branch master updated: [ZEPPELIN-5545] Support multiple maven repo in zeppelin.interpreter.d…
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new f33accd [ZEPPELIN-5545] Support multiple maven repo in zeppelin.interpreter.d… f33accd is described below commit f33accd6190f046f16c5578ff6532c99a27ea8ce Author: huage1994 AuthorDate: Sat Oct 2 23:02:55 2021 +0800 [ZEPPELIN-5545] Support multiple maven repo in zeppelin.interpreter.d… ### What is this PR for? Support multiple maven repo in config item zeppelin.interpreter.dep.mvnRepo ### What type of PR is it? [Improvement] ### Todos * [ ] - Task ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-5545 ### How should this be tested? CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: huage1994 Closes #4239 from huage1994/ZEPPELIN-5545 and squashes the following commits: fc5d917925 [huage1994] [ZEPPELIN-5545] add a test for zeppelin.interpreter.dep.mvnRepo 9161170094 [huage1994] document update for zeppelin.interpreter.dep.mvnRepo b075e3f5ca [huage1994] rename method name `newCentralRepositorie` to `newCentralRepositories` 2b14eef46a [huage1994] [ZEPPELIN-5545] Support multiple maven repo in zeppelin.interpreter.dep.mvnRepo --- docs/setup/operation/configuration.md | 2 +- .../zeppelin/dep/AbstractDependencyResolver.java | 4 +-- .../main/java/org/apache/zeppelin/dep/Booter.java | 36 -- .../java/org/apache/zeppelin/dep/BooterTest.java | 17 ++ .../src/test/resources/zeppelin-site-test.xml | 6 5 files changed, 52 insertions(+), 13 deletions(-) diff --git a/docs/setup/operation/configuration.md b/docs/setup/operation/configuration.md index 9e12345..5627cdd 100644 --- a/docs/setup/operation/configuration.md +++ b/docs/setup/operation/configuration.md @@ -328,7 +328,7 @@ Sources descending by priority: ZEPPELIN_INTERPRETER_DEP_MVNREPO zeppelin.interpreter.dep.mvnRepo -https://repo1.maven.org/maven2/ +https://repo1.maven.org/maven2/,https://repo2.maven.org/maven2/ Remote principal repository for interpreter's additional dependency loading diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java index c678def..623dba7 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java @@ -61,14 +61,14 @@ public abstract class AbstractDependencyResolver { } } session = Booter.newRepositorySystemSession(system, localRepoPath); -repos.add(Booter.newCentralRepository(proxy)); // add maven central +repos.addAll(Booter.newCentralRepositorys(proxy)); // add maven central repos.add(Booter.newLocalRepository()); } public AbstractDependencyResolver(String localRepoPath, Proxy proxy) { this.proxy = proxy; session = Booter.newRepositorySystemSession(system, localRepoPath); -repos.add(Booter.newCentralRepository(proxy)); // add maven central +repos.addAll(Booter.newCentralRepositorys(proxy)); // add maven central repos.add(Booter.newLocalRepository()); } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java index bf5e1da..18cef29 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/Booter.java @@ -30,6 +30,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; /** * Manage mvn repository. @@ -78,20 +82,32 @@ public class Booter { return Paths.get(home).resolve(localRepoPath).toAbsolutePath().toString(); } - public static RemoteRepository newCentralRepository(Proxy proxy) { -String mvnRepo = System.getenv("ZEPPELIN_INTERPRETER_DEP_MVNREPO"); -if (mvnRepo == null) { - mvnRepo = ZeppelinConfiguration.create().getString( + public static List newCentralRepositorys(Proxy proxy) { +String mvnRepoEnv = System.getenv("ZEPPELIN_INTERPRETER_DEP_MVNREPO"); +if (mvnRepoEnv == null) { + mvnRepoEnv = ZeppelinConfiguration.create().getString( ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_DEP_MVNREPO); } -
[zeppelin] branch master updated: ZEPPELIN-5542 Use headless Firefox instead of PhantomJS
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new e29fc93 ZEPPELIN-5542 Use headless Firefox instead of PhantomJS e29fc93 is described below commit e29fc9355f4ab3d1ebc849047f26f366a42c53d5 Author: Martin Tzvetanov Grigorov AuthorDate: Mon Sep 27 14:04:45 2021 +0300 ZEPPELIN-5542 Use headless Firefox instead of PhantomJS ### What is this PR for? Replaces the usage of PhantomJS for the JavaScript tests with headless Firefox. PhantomJS is not maintained since few years. Using a real browser will make the tests more close to the real usage. ### What type of PR is it? Improvement ### What is the Jira issue? * Jira https://issues.apache.org/jira/browse/ZEPPELIN-5542 ### How should this be tested? * `mvn test -Pweb-e2e` should pass for `zeppenlin-web` module ### Questions: * Does the licenses files need update? - NO * Is there breaking changes for older versions? - NO * Does this needs documentation? - NO Author: Martin Tzvetanov Grigorov Closes #4237 from martin-g/zeppelin-5542-use-headless-firefox-for-e2e-tests and squashes the following commits: 554c9f2ce9 [Martin Tzvetanov Grigorov] ZEPPELIN-5542 Use headless Firefox instead of PhantomJS --- zeppelin-web/karma.conf.js | 11 ++- zeppelin-web/package.json | 3 +-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/zeppelin-web/karma.conf.js b/zeppelin-web/karma.conf.js index e122648..018ccb5 100644 --- a/zeppelin-web/karma.conf.js +++ b/zeppelin-web/karma.conf.js @@ -115,7 +115,16 @@ module.exports = function(config) { // - Safari (only Mac) // - PhantomJS // - IE (only Windows) -browsers: [ 'PhantomJS' ], +browsers: [ 'FirefoxHeadless' ], + +plugins: [ + 'karma-coverage', + 'karma-jasmine', + 'karma-sourcemap-loader', + 'karma-webpack', + 'karma-spec-reporter', + 'karma-firefox-launcher', +], reporters: ['spec', 'coverage'], diff --git a/zeppelin-web/package.json b/zeppelin-web/package.json index de83a62..386bd9d 100644 --- a/zeppelin-web/package.json +++ b/zeppelin-web/package.json @@ -21,7 +21,6 @@ "test:watch": "karma start karma.conf.js --single-run=false", "pree2e": "webdriver-manager update --gecko false --versions.chrome=2.35", "e2e": "protractor protractor.conf.js", -"pretest": "npm rebuild phantomjs-prebuilt", "karma-test": "karma start karma.conf.js" }, "dependencies": { @@ -83,9 +82,9 @@ "karma": "~3.1.3", "karma-coverage": "^1.1.2", "karma-jasmine": "~2.0.1", -"karma-phantomjs-launcher": "^1.0.4", "karma-sourcemap-loader": "^0.3.7", "karma-spec-reporter": "0.0.31", +"karma-firefox-launcher": "2.1.1", "karma-webpack": "^3.0.5", "load-grunt-tasks": "^0.4.0", "mini-css-extract-plugin": "^0.4.4",
[zeppelin] branch branch-0.10 updated: [ZEPPELIN-5352] Support flink in k8s mode
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new 76fe06b [ZEPPELIN-5352] Support flink in k8s mode 76fe06b is described below commit 76fe06b5ce09ffbf934b5849fa5f771813ee9967 Author: Jeff Zhang AuthorDate: Fri Oct 8 20:57:29 2021 +0800 [ZEPPELIN-5352] Support flink in k8s mode ### What is this PR for? Flink support k8s native mode from flink 1.11, this PR is to support k8s-application mode. The implementation is very similar as yarn-application mode. ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5352 ### How should this be tested? * Ci pass and manually tested ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4116 from zjffdu/ZEPPELIN-5352 and squashes the following commits: 983a3793da [Jeff Zhang] address comment 11d0c60af1 [Jeff Zhang] [ZEPPELIN-5352]. Support flink in k8s mode (cherry picked from commit 28bc9722700905777f2304a843a126c302d2023e) Signed-off-by: Jeff Zhang --- bin/interpreter.sh | 6 +- ...va => ApplicationModeExecutionEnvironment.java} | 6 +- java => ApplicationModeStreamEnvironment.java} | 8 +-- .../zeppelin/flink/FlinkScalaInterpreter.scala | 32 - .../zeppelin/flink/internal/FlinkILoop.scala | 9 ++- .../zeppelin/flink/internal/FlinkShell.scala | 20 +- .../zeppelin/integration/FlinkIntegrationTest.java | 2 +- .../remote/RemoteInterpreterServer.java| 19 +++-- .../launcher/FlinkInterpreterLauncher.java | 83 +- .../zeppelin/interpreter/InterpreterSetting.java | 4 ++ .../remote/ExecRemoteInterpreterProcess.java | 9 ++- 11 files changed, 148 insertions(+), 50 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index c75a299..6d9c048 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -304,9 +304,9 @@ if [[ -n "${SPARK_SUBMIT}" ]]; then else INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}" "--driver-class-path" "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}" "--driver-java-options" "${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}" "${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") fi -elif [[ "${ZEPPELIN_FLINK_YARN_APPLICATION}" == "true" ]]; then - IFS='|' read -r -a ZEPPELIN_FLINK_YARN_APPLICATION_CONF_ARRAY <<< "${ZEPPELIN_FLINK_YARN_APPLICATION_CONF}" - INTERPRETER_RUN_COMMAND+=("${FLINK_HOME}/bin/flink" "run-application" "-c" "${ZEPPELIN_SERVER}" "-t" "yarn-application" "${ZEPPELIN_FLINK_YARN_APPLICATION_CONF_ARRAY[@]}" "${FLINK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") +elif [[ -n "${ZEPPELIN_FLINK_APPLICATION_MODE}" ]]; then + IFS='|' read -r -a ZEPPELIN_FLINK_APPLICATION_MODE_CONF_ARRAY <<< "${ZEPPELIN_FLINK_APPLICATION_MODE_CONF}" + INTERPRETER_RUN_COMMAND+=("${FLINK_HOME}/bin/flink" "run-application" "-c" "${ZEPPELIN_SERVER}" "-t" "${ZEPPELIN_FLINK_APPLICATION_MODE}" "${ZEPPELIN_FLINK_APPLICATION_MODE_CONF_ARRAY[@]}" "${FLINK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") else IFS=' ' read -r -a JAVA_INTP_OPTS_ARRAY <<< "${JAVA_INTP_OPTS}" IFS=' ' read -r -a ZEPPELIN_INTP_MEM_ARRAY <<< "${ZEPPELIN_INTP_MEM}" diff --git a/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/YarnApplicationExecutionEnvironment.java b/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/ApplicationModeExecutionEnvironment.java similarity index 94% rename from flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/YarnApplicationExecutionEnvironment.java rename to flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/ApplicationModeExecutionEnvironment.java index 2cfd0e3..33afa03 100644 --- a/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/YarnApplicationExecutionEnvironment.java +++ b/
[zeppelin] branch master updated: [ZEPPELIN-5352] Support flink in k8s mode
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 28bc972 [ZEPPELIN-5352] Support flink in k8s mode 28bc972 is described below commit 28bc9722700905777f2304a843a126c302d2023e Author: Jeff Zhang AuthorDate: Fri Oct 8 20:57:29 2021 +0800 [ZEPPELIN-5352] Support flink in k8s mode ### What is this PR for? Flink support k8s native mode from flink 1.11, this PR is to support k8s-application mode. The implementation is very similar as yarn-application mode. ### What type of PR is it? [Feature] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5352 ### How should this be tested? * Ci pass and manually tested ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4116 from zjffdu/ZEPPELIN-5352 and squashes the following commits: 983a3793da [Jeff Zhang] address comment 11d0c60af1 [Jeff Zhang] [ZEPPELIN-5352]. Support flink in k8s mode --- bin/interpreter.sh | 6 +- ...va => ApplicationModeExecutionEnvironment.java} | 6 +- java => ApplicationModeStreamEnvironment.java} | 8 +-- .../zeppelin/flink/FlinkScalaInterpreter.scala | 32 - .../zeppelin/flink/internal/FlinkILoop.scala | 9 ++- .../zeppelin/flink/internal/FlinkShell.scala | 20 +- .../zeppelin/integration/FlinkIntegrationTest.java | 2 +- .../remote/RemoteInterpreterServer.java| 19 +++-- .../launcher/FlinkInterpreterLauncher.java | 83 +- .../zeppelin/interpreter/InterpreterSetting.java | 4 ++ .../remote/ExecRemoteInterpreterProcess.java | 9 ++- 11 files changed, 148 insertions(+), 50 deletions(-) diff --git a/bin/interpreter.sh b/bin/interpreter.sh index c75a299..6d9c048 100755 --- a/bin/interpreter.sh +++ b/bin/interpreter.sh @@ -304,9 +304,9 @@ if [[ -n "${SPARK_SUBMIT}" ]]; then else INTERPRETER_RUN_COMMAND+=("${SPARK_SUBMIT}" "--class" "${ZEPPELIN_SERVER}" "--driver-class-path" "${ZEPPELIN_INTP_CLASSPATH_OVERRIDES}:${ZEPPELIN_INTP_CLASSPATH}" "--driver-java-options" "${JAVA_INTP_OPTS}" "${SPARK_SUBMIT_OPTIONS_ARRAY[@]}" "${ZEPPELIN_SPARK_CONF_ARRAY[@]}" "${SPARK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") fi -elif [[ "${ZEPPELIN_FLINK_YARN_APPLICATION}" == "true" ]]; then - IFS='|' read -r -a ZEPPELIN_FLINK_YARN_APPLICATION_CONF_ARRAY <<< "${ZEPPELIN_FLINK_YARN_APPLICATION_CONF}" - INTERPRETER_RUN_COMMAND+=("${FLINK_HOME}/bin/flink" "run-application" "-c" "${ZEPPELIN_SERVER}" "-t" "yarn-application" "${ZEPPELIN_FLINK_YARN_APPLICATION_CONF_ARRAY[@]}" "${FLINK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") +elif [[ -n "${ZEPPELIN_FLINK_APPLICATION_MODE}" ]]; then + IFS='|' read -r -a ZEPPELIN_FLINK_APPLICATION_MODE_CONF_ARRAY <<< "${ZEPPELIN_FLINK_APPLICATION_MODE_CONF}" + INTERPRETER_RUN_COMMAND+=("${FLINK_HOME}/bin/flink" "run-application" "-c" "${ZEPPELIN_SERVER}" "-t" "${ZEPPELIN_FLINK_APPLICATION_MODE}" "${ZEPPELIN_FLINK_APPLICATION_MODE_CONF_ARRAY[@]}" "${FLINK_APP_JAR}" "${CALLBACK_HOST}" "${PORT}" "${INTP_GROUP_ID}" "${INTP_PORT}") else IFS=' ' read -r -a JAVA_INTP_OPTS_ARRAY <<< "${JAVA_INTP_OPTS}" IFS=' ' read -r -a ZEPPELIN_INTP_MEM_ARRAY <<< "${ZEPPELIN_INTP_MEM}" diff --git a/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/YarnApplicationExecutionEnvironment.java b/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/ApplicationModeExecutionEnvironment.java similarity index 94% rename from flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/YarnApplicationExecutionEnvironment.java rename to flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/ApplicationModeExecutionEnvironment.java index 2cfd0e3..33afa03 100644 --- a/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/YarnApplicationExecutionEnvironment.java +++ b/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/ApplicationModeExecutionEnvironment
[zeppelin] branch branch-0.10 updated: [HotFix] Fix flink1.14-shims version
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new be8cc50 [HotFix] Fix flink1.14-shims version be8cc50 is described below commit be8cc5009bd680754fca7b853540e181f76b0680 Author: Jeff Zhang AuthorDate: Fri Oct 8 10:26:31 2021 +0800 [HotFix] Fix flink1.14-shims version --- flink/flink1.14-shims/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flink/flink1.14-shims/pom.xml b/flink/flink1.14-shims/pom.xml index c160548..faa0c1a 100644 --- a/flink/flink1.14-shims/pom.xml +++ b/flink/flink1.14-shims/pom.xml @@ -21,14 +21,14 @@ flink-parent org.apache.zeppelin -0.11.0-SNAPSHOT +0.10.1-SNAPSHOT ../pom.xml 4.0.0 org.apache.zeppelin flink1.14-shims -0.11.0-SNAPSHOT +0.10.1-SNAPSHOT jar Zeppelin: Flink1.14 Shims @@ -196,4 +196,4 @@ - \ No newline at end of file +
[zeppelin] branch branch-0.10 updated: [ZEPPELIN-5547] Rename note doesn't change the name field in file json
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new f51531a [ZEPPELIN-5547] Rename note doesn't change the name field in file json f51531a is described below commit f51531aba8838ef1e1f158731ab9232a24bb9a75 Author: Jeff Zhang AuthorDate: Mon Oct 4 20:36:11 2021 +0800 [ZEPPELIN-5547] Rename note doesn't change the name field in file json ### What is this PR for? This PR would update note file if the file name is changed. Otherwise the `name` field in note json would be inconsistent with note file name. They would become consistent when user do save note operation afterwards. ### What type of PR is it? [Bug Fix ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5547 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4240 from zjffdu/ZEPPELIN-5547 and squashes the following commits: 84eb8cd336 [Jeff Zhang] address comments 92e0c73bf0 [Jeff Zhang] [ZEPPELIN-5547] Rename note doesn't change the name field in file json (cherry picked from commit a0c3e5da00c0a8c51db527b27f6ec673d450ef14) Signed-off-by: Jeff Zhang --- .../org/apache/zeppelin/notebook/NoteManager.java | 7 +++ .../org/apache/zeppelin/notebook/NotebookTest.java | 22 ++ 2 files changed, 29 insertions(+) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java index 484ee09..85e318d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java @@ -246,6 +246,13 @@ public class NoteManager { // update notebookrepo this.notebookRepo.move(noteId, notePath, newNotePath, subject); + +// save note if note name is changed, because we need to update the note field in note json. +String oldNoteName = getNoteName(notePath); +String newNoteName = getNoteName(newNotePath); +if (!StringUtils.equalsIgnoreCase(oldNoteName, newNoteName)) { + this.notebookRepo.save(noteNode.note, subject); +} } public void moveFolder(String folderPath, diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 1f5fb70..63a9d4d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.notebook; +import org.apache.commons.io.IOUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObjectRegistry; @@ -49,6 +50,7 @@ import org.slf4j.LoggerFactory; import org.eclipse.aether.RepositoryException; import java.io.File; +import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; @@ -1582,6 +1584,26 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo } } + @Test + public void testMoveNote() throws InterruptedException, IOException { +Note note = null; +try { + note = notebook.createNote("note1", anonymous); + assertEquals("note1", note.getName()); + assertEquals("/note1", note.getPath()); + notebook.moveNote(note.getId(), "/tmp/note2", anonymous); + + // read note json file to check the name field is updated + File noteFile = new File(conf.getNotebookDir() + "/" + notebookRepo.buildNoteFileName(note)); + String noteJson = IOUtils.toString(new FileInputStream(noteFile)); + assertTrue(noteJson, noteJson.contains("note2")); +} finally { + if (note != null) { +notebook.removeNote(note, anonymous); + } +} + } + private void delete(File file) { if (file.isFile()) { file.delete();
[zeppelin] branch master updated: [ZEPPELIN-5547] Rename note doesn't change the name field in file json
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new a0c3e5d [ZEPPELIN-5547] Rename note doesn't change the name field in file json a0c3e5d is described below commit a0c3e5da00c0a8c51db527b27f6ec673d450ef14 Author: Jeff Zhang AuthorDate: Mon Oct 4 20:36:11 2021 +0800 [ZEPPELIN-5547] Rename note doesn't change the name field in file json ### What is this PR for? This PR would update note file if the file name is changed. Otherwise the `name` field in note json would be inconsistent with note file name. They would become consistent when user do save note operation afterwards. ### What type of PR is it? [Bug Fix ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5547 ### How should this be tested? * CI pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4240 from zjffdu/ZEPPELIN-5547 and squashes the following commits: 84eb8cd336 [Jeff Zhang] address comments 92e0c73bf0 [Jeff Zhang] [ZEPPELIN-5547] Rename note doesn't change the name field in file json --- .../org/apache/zeppelin/notebook/NoteManager.java | 7 +++ .../org/apache/zeppelin/notebook/NotebookTest.java | 22 ++ 2 files changed, 29 insertions(+) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java index 484ee09..85e318d 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java @@ -246,6 +246,13 @@ public class NoteManager { // update notebookrepo this.notebookRepo.move(noteId, notePath, newNotePath, subject); + +// save note if note name is changed, because we need to update the note field in note json. +String oldNoteName = getNoteName(notePath); +String newNoteName = getNoteName(newNotePath); +if (!StringUtils.equalsIgnoreCase(oldNoteName, newNoteName)) { + this.notebookRepo.save(noteNode.note, subject); +} } public void moveFolder(String folderPath, diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 1f5fb70..63a9d4d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.notebook; +import org.apache.commons.io.IOUtils; import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObjectRegistry; @@ -49,6 +50,7 @@ import org.slf4j.LoggerFactory; import org.eclipse.aether.RepositoryException; import java.io.File; +import java.io.FileInputStream; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; @@ -1582,6 +1584,26 @@ public class NotebookTest extends AbstractInterpreterTest implements ParagraphJo } } + @Test + public void testMoveNote() throws InterruptedException, IOException { +Note note = null; +try { + note = notebook.createNote("note1", anonymous); + assertEquals("note1", note.getName()); + assertEquals("/note1", note.getPath()); + notebook.moveNote(note.getId(), "/tmp/note2", anonymous); + + // read note json file to check the name field is updated + File noteFile = new File(conf.getNotebookDir() + "/" + notebookRepo.buildNoteFileName(note)); + String noteJson = IOUtils.toString(new FileInputStream(noteFile)); + assertTrue(noteJson, noteJson.contains("note2")); +} finally { + if (note != null) { +notebook.removeNote(note, anonymous); + } +} + } + private void delete(File file) { if (file.isFile()) { file.delete();
[zeppelin] branch branch-0.10 updated: [ZEPPELIN-5510]Thread leakage in JobManager of flink interpreter.
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new 3990ce5 [ZEPPELIN-5510]Thread leakage in JobManager of flink interpreter. 3990ce5 is described below commit 3990ce5efd7f1a0cf709854af627ab25dce36f2d Author: nicolasgong AuthorDate: Mon Sep 27 12:43:57 2021 +0800 [ZEPPELIN-5510]Thread leakage in JobManager of flink interpreter. [ZEPPELIN-5510] Thread leakage in JobManager of flink interpreter. ### What is this PR for? There are some logic problems, which may lead to thread leaks. Fix code issues. ### What type of PR is it? [Improvement] ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-5510 Author: nicolasgong Author: YiningGong Closes #4216 from NicolasGong/bugfix-0.2 and squashes the following commits: 57835a8985 [nicolasgong] Added null - value verification for parameters a9bb128c22 [YiningGong] Thread leakage in JobManager of flink interpreter. (cherry picked from commit 79750a88025bd64cff5830cf2ec8e6a6181454f5) Signed-off-by: Jeff Zhang --- .../main/java/org/apache/zeppelin/flink/JobManager.java | 17 - 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/JobManager.java b/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/JobManager.java index d60ff1a..49c803a 100644 --- a/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/JobManager.java +++ b/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/JobManager.java @@ -62,19 +62,20 @@ public class JobManager { LOGGER.info("Creating JobManager at flinkWebUrl: {}, displayedFlinkWebUrl: {}", flinkWebUrl, displayedFlinkWebUrl); } - + public void addJob(InterpreterContext context, JobClient jobClient) { String paragraphId = context.getParagraphId(); JobClient previousJobClient = this.jobs.put(paragraphId, jobClient); +if (previousJobClient != null) { + LOGGER.warn("There's another Job {} that is associated with paragraph {}", + jobClient.getJobID(), paragraphId); + return; +} long checkInterval = Long.parseLong(properties.getProperty("zeppelin.flink.job.check_interval", "1000")); FlinkJobProgressPoller thread = new FlinkJobProgressPoller(flinkWebUrl, jobClient.getJobID(), context, checkInterval); thread.setName("JobProgressPoller-Thread-" + paragraphId); thread.start(); this.jobProgressPollerMap.put(jobClient.getJobID(), thread); -if (previousJobClient != null) { - LOGGER.warn("There's another Job {} that is associated with paragraph {}", - jobClient.getJobID(), paragraphId); -} } public void removeJob(String paragraphId) { @@ -87,6 +88,12 @@ public class JobManager { } FlinkJobProgressPoller jobProgressPoller = this.jobProgressPollerMap.remove(jobClient.getJobID()); +if (jobProgressPoller == null) { +LOGGER.warn("Unable to remove poller, because no poller is associated with paragraph: " ++ paragraphId); +return; +} + jobProgressPoller.cancel(); jobProgressPoller.interrupt(); }
[zeppelin] branch master updated: [ZEPPELIN-5510]Thread leakage in JobManager of flink interpreter.
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new 79750a8 [ZEPPELIN-5510]Thread leakage in JobManager of flink interpreter. 79750a8 is described below commit 79750a88025bd64cff5830cf2ec8e6a6181454f5 Author: nicolasgong AuthorDate: Mon Sep 27 12:43:57 2021 +0800 [ZEPPELIN-5510]Thread leakage in JobManager of flink interpreter. [ZEPPELIN-5510] Thread leakage in JobManager of flink interpreter. ### What is this PR for? There are some logic problems, which may lead to thread leaks. Fix code issues. ### What type of PR is it? [Improvement] ### What is the Jira issue? https://issues.apache.org/jira/browse/ZEPPELIN-5510 Author: nicolasgong Author: YiningGong Closes #4216 from NicolasGong/bugfix-0.2 and squashes the following commits: 57835a8985 [nicolasgong] Added null - value verification for parameters a9bb128c22 [YiningGong] Thread leakage in JobManager of flink interpreter. --- .../main/java/org/apache/zeppelin/flink/JobManager.java | 17 - 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/JobManager.java b/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/JobManager.java index d60ff1a..49c803a 100644 --- a/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/JobManager.java +++ b/flink/flink-scala-parent/src/main/java/org/apache/zeppelin/flink/JobManager.java @@ -62,19 +62,20 @@ public class JobManager { LOGGER.info("Creating JobManager at flinkWebUrl: {}, displayedFlinkWebUrl: {}", flinkWebUrl, displayedFlinkWebUrl); } - + public void addJob(InterpreterContext context, JobClient jobClient) { String paragraphId = context.getParagraphId(); JobClient previousJobClient = this.jobs.put(paragraphId, jobClient); +if (previousJobClient != null) { + LOGGER.warn("There's another Job {} that is associated with paragraph {}", + jobClient.getJobID(), paragraphId); + return; +} long checkInterval = Long.parseLong(properties.getProperty("zeppelin.flink.job.check_interval", "1000")); FlinkJobProgressPoller thread = new FlinkJobProgressPoller(flinkWebUrl, jobClient.getJobID(), context, checkInterval); thread.setName("JobProgressPoller-Thread-" + paragraphId); thread.start(); this.jobProgressPollerMap.put(jobClient.getJobID(), thread); -if (previousJobClient != null) { - LOGGER.warn("There's another Job {} that is associated with paragraph {}", - jobClient.getJobID(), paragraphId); -} } public void removeJob(String paragraphId) { @@ -87,6 +88,12 @@ public class JobManager { } FlinkJobProgressPoller jobProgressPoller = this.jobProgressPollerMap.remove(jobClient.getJobID()); +if (jobProgressPoller == null) { +LOGGER.warn("Unable to remove poller, because no poller is associated with paragraph: " ++ paragraphId); +return; +} + jobProgressPoller.cancel(); jobProgressPoller.interrupt(); }
[zeppelin] branch branch-0.10 updated: [ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch branch-0.10 in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/branch-0.10 by this push: new be57acc [ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8 be57acc is described below commit be57accef8452da5974b8a5529ffa8b52223921e Author: Jeff Zhang AuthorDate: Thu Sep 16 16:41:46 2021 +0800 [ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8 ### What is this PR for? The root cause is due to ast api is changed after python 3.8, see https://github.com/ipython/ipython/pull/11593. * The PR fixed this issue in `zeppelin_python.py`. * This PR add tests for Python 3.8 for both python interpreter & spark interpreter (spark 2.4 doesn't support python 3.8, so there's no python 3.8 test for spark 2.4). `jupyter_client` 5 is used because jupyter_client 7 doesn't work (tracked in ZEPPELIN-5533) * bokeh test is disabled for flink interpreter. Because pyflink's dependencies conflicts with bokeh2. ### What type of PR is it? [Bug Fix ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5525 ### How should this be tested? * Ci pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4230 from zjffdu/ZEPPELIN-5525 and squashes the following commits: b816a530c0 [Jeff Zhang] [ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8 (cherry picked from commit fa1f72891f7c83911deefedef5275fa766b5b6ab) Signed-off-by: Jeff Zhang --- .github/workflows/core.yml | 31 +++--- .../zeppelin/flink/IPyFlinkInterpreterTest.java| 6 .../src/main/resources/python/zeppelin_python.py | 13 ++-- .../zeppelin/python/IPythonInterpreterTest.java| 24 ++ testing/env_python_3.7_with_R.yml | 33 +++ testing/env_python_3.8_with_R.yml | 33 +++ testing/env_python_3_with_R.yml| 37 +++--- testing/env_python_3_with_R_and_tensorflow.yml | 36 ++--- testing/env_python_3_with_flink_110.yml| 34 ++-- testing/env_python_3_with_flink_111.yml| 34 ++-- testing/env_python_3_with_flink_112.yml| 37 +++--- testing/env_python_3_with_flink_113.yml| 37 +++--- 12 files changed, 231 insertions(+), 124 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 81e0822..3f866fd 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -32,6 +32,7 @@ jobs: fail-fast: false matrix: hadoop: [hadoop2, hadoop3] +python: [3.7, 3.8] steps: - name: Checkout uses: actions/checkout@v2 @@ -51,12 +52,12 @@ jobs: key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-zeppelin- - - name: Setup conda environment with python 3.7 and R + - name: Setup conda environment with python ${{ matrix.python }} and R uses: conda-incubator/setup-miniconda@v2 with: activate-environment: python_3_with_R - environment-file: testing/env_python_3_with_R.yml - python-version: 3.7 + environment-file: testing/env_python_${{ matrix.python }}_with_R.yml + python-version: ${{ matrix.python }} auto-activate-base: false channel-priority: strict - name: Make IRkernel available to Jupyter @@ -298,6 +299,10 @@ jobs: spark-3-0-and-scala-2-12-and-other-interpreter: runs-on: ubuntu-20.04 +strategy: + fail-fast: false + matrix: +python: [ 3.7, 3.8 ] steps: - name: Checkout uses: actions/checkout@v2 @@ -317,12 +322,12 @@ jobs: key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-zeppelin- - - name: Setup conda environment with python 3.7 and R + - name: Setup conda environment with python ${{ matrix.python }} and R uses: conda-incubator/setup-miniconda@v2 with: activate-environment: python_3_with_R - environment-file: testing/env_python_3_with_R.yml - python-version: 3.7 + environment-file: testing/env_python_${{ matrix.python }}_with_R.yml + python-version: ${{ matrix.python }} auto-activate-base: false - name: Make IRkernel available to Jup
[zeppelin] branch master updated: [ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8
This is an automated email from the ASF dual-hosted git repository. zjffdu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/zeppelin.git The following commit(s) were added to refs/heads/master by this push: new fa1f728 [ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8 fa1f728 is described below commit fa1f72891f7c83911deefedef5275fa766b5b6ab Author: Jeff Zhang AuthorDate: Thu Sep 16 16:41:46 2021 +0800 [ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8 ### What is this PR for? The root cause is due to ast api is changed after python 3.8, see https://github.com/ipython/ipython/pull/11593. * The PR fixed this issue in `zeppelin_python.py`. * This PR add tests for Python 3.8 for both python interpreter & spark interpreter (spark 2.4 doesn't support python 3.8, so there's no python 3.8 test for spark 2.4). `jupyter_client` 5 is used because jupyter_client 7 doesn't work (tracked in ZEPPELIN-5533) * bokeh test is disabled for flink interpreter. Because pyflink's dependencies conflicts with bokeh2. ### What type of PR is it? [Bug Fix ] ### Todos * [ ] - Task ### What is the Jira issue? * https://issues.apache.org/jira/browse/ZEPPELIN-5525 ### How should this be tested? * Ci pass ### Screenshots (if appropriate) ### Questions: * Does the licenses files need update? No * Is there breaking changes for older versions? No * Does this needs documentation? No Author: Jeff Zhang Closes #4230 from zjffdu/ZEPPELIN-5525 and squashes the following commits: b816a530c0 [Jeff Zhang] [ZEPPELIN-5525] Python vanillar interpreter doesn't' work in Python 3.8 --- .github/workflows/core.yml | 31 +++--- .../zeppelin/flink/IPyFlinkInterpreterTest.java| 6 .../src/main/resources/python/zeppelin_python.py | 13 ++-- .../zeppelin/python/IPythonInterpreterTest.java| 24 ++ testing/env_python_3.7_with_R.yml | 33 +++ testing/env_python_3.8_with_R.yml | 33 +++ testing/env_python_3_with_R.yml| 37 +++--- testing/env_python_3_with_R_and_tensorflow.yml | 36 ++--- testing/env_python_3_with_flink_110.yml| 34 ++-- testing/env_python_3_with_flink_111.yml| 34 ++-- testing/env_python_3_with_flink_112.yml| 37 +++--- testing/env_python_3_with_flink_113.yml| 37 +++--- 12 files changed, 231 insertions(+), 124 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 154cd24..f65e832 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -32,6 +32,7 @@ jobs: fail-fast: false matrix: hadoop: [hadoop2, hadoop3] +python: [3.7, 3.8] steps: - name: Checkout uses: actions/checkout@v2 @@ -51,12 +52,12 @@ jobs: key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-zeppelin- - - name: Setup conda environment with python 3.7 and R + - name: Setup conda environment with python ${{ matrix.python }} and R uses: conda-incubator/setup-miniconda@v2 with: activate-environment: python_3_with_R - environment-file: testing/env_python_3_with_R.yml - python-version: 3.7 + environment-file: testing/env_python_${{ matrix.python }}_with_R.yml + python-version: ${{ matrix.python }} auto-activate-base: false channel-priority: strict - name: Make IRkernel available to Jupyter @@ -298,6 +299,10 @@ jobs: spark-3-0-and-scala-2-12-and-other-interpreter: runs-on: ubuntu-20.04 +strategy: + fail-fast: false + matrix: +python: [ 3.7, 3.8 ] steps: - name: Checkout uses: actions/checkout@v2 @@ -317,12 +322,12 @@ jobs: key: ${{ runner.os }}-zeppelin-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-zeppelin- - - name: Setup conda environment with python 3.7 and R + - name: Setup conda environment with python ${{ matrix.python }} and R uses: conda-incubator/setup-miniconda@v2 with: activate-environment: python_3_with_R - environment-file: testing/env_python_3_with_R.yml - python-version: 3.7 + environment-file: testing/env_python_${{ matrix.python }}_with_R.yml + python-version: ${{ matrix.python }} auto-activate-base: false - name: Make IRkernel available to Jupyter run: | @@ -330,11 +335,15 @@ jobs: - name: install environment run: | mvn ins