This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new bc8cf1a ZEPPELIN-3970. JdbcInterpreter is broken
bc8cf1a is described below
commit bc8cf1ad4e8a903f71530a6feaf2f8bdd04e3156
Author: Jeff Zhang <zjf...@apache.org>
AuthorDate: Mon Jan 28 15:21:26 2019 +0800
ZEPPELIN-3970. JdbcInterpreter is broken
### What is this PR for?
This PR fix the issue of JdbcInterpreter broken issue. This root cause is
that commons-lang is not packaged into the jdbc interpreter jar. I add it into
pom file and also add system test for jdbc interpreter to avoid regression
issue in future
### What type of PR is it?
[Bug Fix ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://jira.apache.org/jira/browse/ZEPPELIN-3970
### How should this be tested?
System test is added in `JdbcIntegrationTest`
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zjf...@apache.org>
Closes #3292 from zjffdu/ZEPPELIN-3970 and squashes the following commits:
8be522830 [Jeff Zhang] ZEPPELIN-3970. JdbcInterpreter is broken
---
.travis.yml | 6 +-
jdbc/pom.xml | 13 +++-
.../org/apache/zeppelin/jdbc/JDBCInterpreter.java | 14 ++--
.../org/apache/zeppelin/jdbc/SqlCompleter.java | 4 +-
.../remote/RemoteInterpreterServerTest.java | 1 +
.../apache/zeppelin/service/NotebookService.java | 3 +-
.../zeppelin/rest/ZeppelinSparkClusterTest.java | 2 +-
.../zeppelin/interpreter/InterpreterSetting.java | 21 ++++--
.../interpreter/InterpreterSettingManager.java | 56 ++++++++--------
.../java/org/apache/zeppelin/notebook/Note.java | 13 ++--
.../zeppelin/interpreter/JdbcIntegrationTest.java | 75 ++++++++++++++++++++++
.../src/test/resources/log4j.properties | 6 +-
12 files changed, 156 insertions(+), 58 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index ada776e..9654a94 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -70,7 +70,7 @@ matrix:
- sudo: required
jdk: "oraclejdk8"
dist: trusty
- env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Phelium-dev
-Pexamples -Pscala-2.11" BUILD_FLAG="install -Pbuild-distr -DskipRat"
TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl
${INTERPRETERS}"
TEST_PROJECTS="-Dtests.to.exclude=**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java
-DfailIfNoTests=false"
+ env: PYTHON="3" SPARKR="true" PROFILE="-Pspark-2.2 -Phelium-dev
-Pexamples -Pscala-2.11" BUILD_FLAG="install -Pbuild-distr -DskipRat"
TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl
${INTERPRETERS}"
TEST_PROJECTS="-Dtests.to.exclude=**/JdbcIntegrationTest.java,**/SparkIntegrationTest.java,**/ZeppelinSparkClusterTest.java,**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java
-DfailIfNoTests=false"
# Test selenium with spark module for 1.6.3
- jdk: "oraclejdk8"
@@ -98,11 +98,11 @@ matrix:
dist: trusty
env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.10"
PROFILE="-Pspark-1.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install
-DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl
zeppelin-zengine,spark/interpreter,spark/spark-dependencies"
TEST_PROJECTS="-Dtest=SparkIntegrationTestPt1,org.apache.zeppelin.spark.*
-DfailIfNoTests=false"
- # Integration test of spark interpreter with different spark versions
under python3, only run SparkIntegrationTestPt2. Also run spark unit test of
spark 1.6 in this build.
+ # Integration test of spark interpreter with different spark versions
under python3, only run SparkIntegrationTestPt2. Also run spark unit test of
spark 1.6 in this build. And run JdbcIntegrationTest here as well
- sudo: required
jdk: "oraclejdk8"
dist: trusty
- env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.10"
PROFILE="-Pspark-1.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install
-DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl
zeppelin-zengine,spark/interpreter,spark/spark-dependencies"
TEST_PROJECTS="-Dtest=SparkIntegrationTestPt2,org.apache.zeppelin.spark.*
-DfailIfNoTests=false"
+ env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.10"
PROFILE="-Pspark-1.6 -Pscala-2.10" SPARKR="true" BUILD_FLAG="install
-DskipTests -DskipRat -am" TEST_FLAG="test -DskipRat -am" MODULES="-pl
jdbc,zeppelin-zengine,spark/interpreter,spark/spark-dependencies"
TEST_PROJECTS="-Dtest=JdbcIntegrationTest,SparkIntegrationTestPt2,org.apache.zeppelin.spark.*
-DfailIfNoTests=false"
# Test spark module for 2.4.0 with scala 2.11
- jdk: "oraclejdk8"
diff --git a/jdbc/pom.xml b/jdbc/pom.xml
index eff4f50..3a8b592 100644
--- a/jdbc/pom.xml
+++ b/jdbc/pom.xml
@@ -227,7 +227,7 @@
</dependencies>
</profile>
</profiles>
-
+
<properties>
<!--library versions-->
<interpreter.name>jdbc</interpreter.name>
@@ -235,13 +235,14 @@
<hadoop.common.version>2.7.2</hadoop.common.version>
<h2.version>1.4.190</h2.version>
<commons.dbcp2.version>2.0.1</commons.dbcp2.version>
+ <commons-lang3.version>3.7</commons-lang3.version>
<!--test library versions-->
<mockrunner.jdbc.version>1.0.8</mockrunner.jdbc.version>
</properties>
-
+
<dependencies>
-
+
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
@@ -257,6 +258,12 @@
<dependency>
<groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>${commons-lang3.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.commons</groupId>
<artifactId>commons-dbcp2</artifactId>
<version>${commons.dbcp2.version}</version>
</dependency>
diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
index 4d25834..dd361f5 100644
--- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
+++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/JDBCInterpreter.java
@@ -14,18 +14,18 @@
*/
package org.apache.zeppelin.jdbc;
-import static org.apache.commons.lang.StringUtils.containsIgnoreCase;
-import static org.apache.commons.lang.StringUtils.isEmpty;
-import static org.apache.commons.lang.StringUtils.isNotEmpty;
+import static org.apache.commons.lang3.StringUtils.containsIgnoreCase;
+import static org.apache.commons.lang3.StringUtils.isEmpty;
+import static org.apache.commons.lang3.StringUtils.isNotEmpty;
import static
org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod.KERBEROS;
import org.apache.commons.dbcp2.ConnectionFactory;
import org.apache.commons.dbcp2.DriverManagerConnectionFactory;
import org.apache.commons.dbcp2.PoolableConnectionFactory;
import org.apache.commons.dbcp2.PoolingDriver;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.exception.ExceptionUtils;
-import org.apache.commons.lang.mutable.MutableBoolean;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.exception.ExceptionUtils;
+import org.apache.commons.lang3.mutable.MutableBoolean;
import org.apache.commons.pool2.ObjectPool;
import org.apache.commons.pool2.impl.GenericObjectPool;
import org.apache.hadoop.conf.Configuration;
@@ -727,7 +727,7 @@ public class JDBCInterpreter extends KerberosInterpreter {
String statementPrecode =
getProperty(String.format(STATEMENT_PRECODE_KEY_TEMPLATE,
propertyKey));
-
+
if (StringUtils.isNotBlank(statementPrecode)) {
statement.execute(statementPrecode);
}
diff --git a/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
b/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
index 9f52ecb..56cb5cc 100644
--- a/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
+++ b/jdbc/src/main/java/org/apache/zeppelin/jdbc/SqlCompleter.java
@@ -4,8 +4,8 @@ package org.apache.zeppelin.jdbc;
* This source file is based on code taken from SQLLine 1.0.2 See SQLLine
notice in LICENSE
*/
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --git
a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java
b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java
index 9719717..7beeee8 100644
---
a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java
+++
b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterServerTest.java
@@ -130,6 +130,7 @@ public class RemoteInterpreterServerTest {
intpContext.setParagraphId("paragraph_1");
intpContext.setGui("{}");
intpContext.setNoteGui("{}");
+ intpContext.setLocalProperties(new HashMap<>());
// single output of SUCCESS
RemoteInterpreterResult result = server.interpret("session_1",
Test1Interpreter.class.getName(),
diff --git
a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java
b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java
index d7dd33e..62973ff 100644
---
a/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java
+++
b/zeppelin-server/src/main/java/org/apache/zeppelin/service/NotebookService.java
@@ -820,7 +820,8 @@ public class NotebookService {
}
try {
- List<InterpreterCompletion> completions = note.completion(paragraphId,
buffer, cursor);
+ List<InterpreterCompletion> completions = note.completion(paragraphId,
buffer, cursor,
+ context.getAutheInfo());
callback.onSuccess(completions, context);
return completions;
} catch (RuntimeException e) {
diff --git
a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
index d3516bd..dcd79d5 100644
---
a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
+++
b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java
@@ -189,7 +189,7 @@ public class ZeppelinSparkClusterTest extends
AbstractTestRestApi {
assertEquals("2", p.getReturn().message().get(0).getData());
// test code completion
- List<InterpreterCompletion> completions = note.completion(p.getId(),
"sc.", 2);
+ List<InterpreterCompletion> completions = note.completion(p.getId(),
"sc.", 2, AuthenticationInfo.ANONYMOUS);
assertTrue(completions.size() > 0);
// test cancel
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
index 95530b0..7fc0d53 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java
@@ -932,10 +932,23 @@ public class InterpreterSetting {
throw new RuntimeException("Can not convert this type: " +
properties.getClass());
}
- public void waitForReady() throws InterruptedException {
- while (getStatus().equals(
-
org.apache.zeppelin.interpreter.InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES))
{
- Thread.sleep(200);
+ public void waitForReady(long timeout) throws InterpreterException {
+ long start = System.currentTimeMillis();
+ while(status != Status.READY) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ throw new InterpreterException(e);
+ }
+ long now = System.currentTimeMillis();
+ if ((now - start) > timeout) {
+ throw new InterpreterException("Fail to download dependencies in " +
timeout / 1000
+ + " seconds");
+ }
}
}
+
+ public void waitForReady() throws InterpreterException {
+ waitForReady(Long.MAX_VALUE);
+ }
}
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
index d53394d..de2e391 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -633,36 +633,36 @@ public class InterpreterSettingManager implements
NoteEventListener {
*/
private void copyDependenciesFromLocalPath(final InterpreterSetting setting)
{
setting.setStatus(InterpreterSetting.Status.DOWNLOADING_DEPENDENCIES);
- final Thread t = new Thread() {
- public void run() {
- try {
- List<Dependency> deps = setting.getDependencies();
- if (deps != null) {
- for (Dependency d : deps) {
- File destDir = new File(
- conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
-
- int numSplits = d.getGroupArtifactVersion().split(":").length;
- if (!(numSplits >= 3 && numSplits <= 6)) {
-
dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
- new File(destDir, setting.getId()));
- }
+ final Thread t = new Thread() {
+ public void run() {
+ try {
+ List<Dependency> deps = setting.getDependencies();
+ if (deps != null) {
+ for (Dependency d : deps) {
+ File destDir = new File(
+ conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO));
+
+ int numSplits = d.getGroupArtifactVersion().split(":").length;
+ if (!(numSplits >= 3 && numSplits <= 6)) {
+
dependencyResolver.copyLocalDependency(d.getGroupArtifactVersion(),
+ new File(destDir, setting.getId()));
}
}
- setting.setStatus(InterpreterSetting.Status.READY);
- } catch (Exception e) {
- LOGGER.error(String.format("Error while copying deps for
interpreter group : %s," +
- " go to interpreter setting page click on edit and save it
again to make " +
- "this interpreter work properly.",
- setting.getGroup()), e);
- setting.setErrorReason(e.getLocalizedMessage());
- setting.setStatus(InterpreterSetting.Status.ERROR);
- } finally {
-
}
+ setting.setStatus(InterpreterSetting.Status.READY);
+ } catch (Exception e) {
+ LOGGER.error(String.format("Error while copying deps for interpreter
group : %s," +
+ " go to interpreter setting page click on edit and save it
again to make " +
+ "this interpreter work properly.",
+ setting.getGroup()), e);
+ setting.setErrorReason(e.getLocalizedMessage());
+ setting.setStatus(InterpreterSetting.Status.ERROR);
+ } finally {
+
}
- };
- t.start();
+ }
+ };
+ t.start();
}
/**
@@ -795,7 +795,9 @@ public class InterpreterSettingManager implements
NoteEventListener {
}
public void restart(String id) throws InterpreterException {
- interpreterSettings.get(id).close();
+ InterpreterSetting setting = interpreterSettings.get(id);
+ copyDependenciesFromLocalPath(setting);
+ setting.close();
}
public InterpreterSetting get(String id) {
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
index 5bfdb01..a583c82 100644
--- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
+++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java
@@ -311,14 +311,14 @@ public class Note implements JsonSerializable {
Map<String, Input> form = srcParagraph.settings.getForms();
logger.debug("srcParagraph user: " + srcParagraph.getUser());
-
+
newParagraph.setAuthenticationInfo(subject);
newParagraph.setConfig(config);
newParagraph.settings.setParams(param);
newParagraph.settings.setForms(form);
newParagraph.setText(srcParagraph.getText());
newParagraph.setTitle(srcParagraph.getTitle());
-
+
logger.debug("newParagraph user: " + newParagraph.getUser());
try {
@@ -668,10 +668,13 @@ public class Note implements JsonSerializable {
return this.path.startsWith("/" + NoteManager.TRASH_FOLDER);
}
- public List<InterpreterCompletion> completion(String paragraphId, String
buffer, int cursor) {
+ public List<InterpreterCompletion> completion(String paragraphId,
+ String buffer,
+ int cursor,
+ AuthenticationInfo authInfo) {
Paragraph p = getParagraph(paragraphId);
p.setListener(this.paragraphJobListener);
-
+ p.setAuthenticationInfo(authInfo);
return p.completion(buffer, cursor);
}
@@ -688,7 +691,7 @@ public class Note implements JsonSerializable {
if (settings == null || settings.size() == 0) {
return;
}
-
+
for (InterpreterSetting setting : settings) {
InterpreterGroup intpGroup = setting.getInterpreterGroup(user, id);
if (intpGroup != null) {
diff --git
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/JdbcIntegrationTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/JdbcIntegrationTest.java
new file mode 100644
index 0000000..a7a80e9
--- /dev/null
+++
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/JdbcIntegrationTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zeppelin.interpreter;
+
+import com.google.common.collect.Lists;
+import org.apache.zeppelin.dep.Dependency;
+import org.apache.zeppelin.user.AuthenticationInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class JdbcIntegrationTest {
+
+ private static MiniZeppelin zeppelin;
+ private static InterpreterFactory interpreterFactory;
+ private static InterpreterSettingManager interpreterSettingManager;
+
+
+ @BeforeClass
+ public static void setUp() throws IOException {
+ zeppelin = new MiniZeppelin();
+ zeppelin.start();
+ interpreterFactory = zeppelin.getInterpreterFactory();
+ interpreterSettingManager = zeppelin.getInterpreterSettingManager();
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ if (zeppelin != null) {
+ zeppelin.stop();
+ }
+ }
+
+ @Test
+ public void testMySql() throws InterpreterException, InterruptedException {
+ InterpreterSetting interpreterSetting =
interpreterSettingManager.getInterpreterSettingByName("jdbc");
+ interpreterSetting.setProperty("default.driver", "com.mysql.jdbc.Driver");
+ interpreterSetting.setProperty("default.url",
"jdbc:mysql://localhost:3306/");
+ interpreterSetting.setProperty("default.user", "root");
+ Dependency dependency = new
Dependency("mysql:mysql-connector-java:5.1.46");
+ interpreterSetting.setDependencies(Lists.newArrayList(dependency));
+ interpreterSettingManager.restart("jdbc");
+ interpreterSetting.waitForReady(60 * 1000);
+ Interpreter jdbcInterpreter = interpreterFactory.getInterpreter("user1",
"note1", "jdbc", "test");
+ assertNotNull("JdbcInterpreter is null", jdbcInterpreter);
+
+ InterpreterContext context = new InterpreterContext.Builder()
+ .setNoteId("note1")
+ .setParagraphId("paragraph_1")
+ .setAuthenticationInfo(AuthenticationInfo.ANONYMOUS)
+ .build();
+ InterpreterResult interpreterResult = jdbcInterpreter.interpret("show
databases;", context);
+ assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code);
+ }
+}
diff --git a/zeppelin-zengine/src/test/resources/log4j.properties
b/zeppelin-zengine/src/test/resources/log4j.properties
index fd9771c..354a8bc 100644
--- a/zeppelin-zengine/src/test/resources/log4j.properties
+++ b/zeppelin-zengine/src/test/resources/log4j.properties
@@ -43,8 +43,4 @@ log4j.logger.DataNucleus.Datastore=ERROR
log4j.logger.org.hibernate.type=ALL
log4j.logger.org.apache.hadoop=WARN
-log4j.logger.org.apache.zeppelin.plugin=DEBUG
-log4j.logger.org.apache.zeppelin.spark=DEBUG
-log4j.logger.org.apache.zeppelin.python=DEBUG
-
-log4j.logger.org.apache.zeppelin.interpreter=DEBUG
+log4j.logger.org.apache.zeppelin.interpreter.remote.RemoteInterpreterManagedProcess=DEBUG
