This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 5bfe3fc [Fix][SqlTask] fix hive sqltask invalid connection
parameters. (#4917)
5bfe3fc is described below
commit 5bfe3fca673d9f0a0622ef5c3f089e52a9599641
Author: zhuangchong <[email protected]>
AuthorDate: Wed Mar 3 09:15:37 2021 +0800
[Fix][SqlTask] fix hive sqltask invalid connection parameters. (#4917)
* fix hive sqltask invalid connection parameters.
* add setConnParams test method.
* update base datasource test class code style.
* add BaseDataSourceTest in the pom xml.
* update hive conf list jdbc url stitching.
* update hive datasource test class.
---
.../dao/datasource/BaseDataSource.java | 26 +++++++++++++++++
.../dao/datasource/HiveDataSource.java | 17 +++++++++++
.../dao/datasource/BaseDataSourceTest.java | 34 ++++++++++++++++++++++
.../dao/datasource/HiveDataSourceTest.java | 2 +-
.../server/worker/task/sql/SqlTask.java | 3 ++
pom.xml | 1 +
6 files changed, 82 insertions(+), 1 deletion(-)
diff --git
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
index 9bbbf32..e26a375 100644
---
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
+++
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSource.java
@@ -17,12 +17,16 @@
package org.apache.dolphinscheduler.dao.datasource;
+import static org.apache.dolphinscheduler.common.Constants.PASSWORD;
+import static org.apache.dolphinscheduler.common.Constants.USER;
+
import org.apache.dolphinscheduler.common.enums.DbType;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.util.Properties;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -157,6 +161,11 @@ public abstract class BaseDataSource {
separator = ":";
break;
case HIVE:
+ if ("?".equals(otherParams.substring(0, 1))) {
+ break;
+ }
+ separator = ";";
+ break;
case SPARK:
case SQLSERVER:
separator = ";";
@@ -178,6 +187,19 @@ public abstract class BaseDataSource {
return DriverManager.getConnection(getJdbcUrl(), getUser(),
getPassword());
}
+ /**
+ * the data source test connection
+ * @param info Properties
+ * @return Connection Connection
+ * @throws Exception Exception
+ */
+ public Connection getConnection(Properties info) throws Exception {
+ Class.forName(driverClassSelector());
+ info.setProperty(USER, getUser());
+ info.setProperty(PASSWORD, getPassword());
+ return DriverManager.getConnection(getJdbcUrl(), info);
+ }
+
protected String filterOther(String otherParams) {
return otherParams;
}
@@ -226,6 +248,10 @@ public abstract class BaseDataSource {
this.other = other;
}
+ public void setConnParams(String connParams) {
+
+ }
+
public String getJavaSecurityKrb5Conf() {
return javaSecurityKrb5Conf;
}
diff --git
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
index bcf1cdf..8fccb08 100644
---
a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
+++
b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSource.java
@@ -17,13 +17,17 @@
package org.apache.dolphinscheduler.dao.datasource;
+import static org.apache.dolphinscheduler.common.Constants.SEMICOLON;
+
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.DbType;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
import org.apache.dolphinscheduler.common.utils.CommonUtils;
import org.apache.dolphinscheduler.common.utils.HiveConfUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.sql.Connection;
+import java.util.Map;
/**
* data source of hive
@@ -100,4 +104,17 @@ public class HiveDataSource extends BaseDataSource {
return super.getConnection();
}
+ @Override
+ public void setConnParams(String connParams) {
+ // Verification parameters
+ Map<String, String> connParamMap =
CollectionUtils.stringToMap(connParams, SEMICOLON);
+ if (connParamMap.isEmpty()) {
+ return;
+ }
+
+ StringBuilder otherSb = new StringBuilder();
+ connParamMap.forEach((k, v) -> otherSb.append(String.format("%s=%s%s",
k, v, SEMICOLON)));
+ StringBuilder otherAppend = StringUtils.isNotBlank(getOther()) ?
otherSb.append(getOther()) : otherSb.deleteCharAt(otherSb.length() - 1);
+ super.setOther(otherAppend.toString());
+ }
}
diff --git
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java
index 1d02fd4..31c963d 100644
---
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java
+++
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/BaseDataSourceTest.java
@@ -158,4 +158,38 @@ public class BaseDataSourceTest {
}
+ @Test
+ public void testSetConnParams() {
+
+ BaseDataSource hiveDataSource = new HiveDataSource();
+ hiveDataSource.setAddress("jdbc:hive2://127.0.0.1:10000");
+ hiveDataSource.setDatabase("test");
+ hiveDataSource.setPassword("123456");
+ hiveDataSource.setUser("test");
+ hiveDataSource.setConnParams("");
+ Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test",
hiveDataSource.getJdbcUrl());
+
+ //set fake other
+ hiveDataSource.setConnParams("hive.tez.container.size=20000;");
+
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test?hive.tez.container.size=20000",
hiveDataSource.getJdbcUrl());
+
+ hiveDataSource.setOther(null);
+ hiveDataSource.setConnParams("hive.tez.container.size=20000");
+
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test?hive.tez.container.size=20000",
hiveDataSource.getJdbcUrl());
+
+ hiveDataSource.setOther(null);
+
hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
+
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100?hive.tez.container.size=20000",
hiveDataSource.getJdbcUrl());
+
+ hiveDataSource.setOther("charset=UTF-8");
+ Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;charset=UTF-8",
hiveDataSource.getJdbcUrl());
+
+
hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
+
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100;charset=UTF-8?hive.tez.container.size=20000",
hiveDataSource.getJdbcUrl());
+
+ hiveDataSource.setOther("charset=UTF-8;hive.exec.stagingdir=/tmp");
+
hiveDataSource.setConnParams("hive.tez.container.size=20000;hive.zzz=100");
+
Assert.assertEquals("jdbc:hive2://127.0.0.1:10000/test;hive.zzz=100;charset=UTF-8?hive.tez.container.size=20000;hive.exec.stagingdir=/tmp",
hiveDataSource.getJdbcUrl());
+ }
+
}
diff --git
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java
index 3920fd3..145b7f3 100644
---
a/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java
+++
b/dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/datasource/HiveDataSourceTest.java
@@ -81,7 +81,7 @@ public class HiveDataSourceTest {
hiveDataSource.setOther("hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2");
Assert.assertEquals(
-
"jdbc:hive2://127.0.0.1:10000/test;?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
+
"jdbc:hive2://127.0.0.1:10000/test?hive.mapred.mode=strict;hive.server2.thrift.http.path=hs2",
hiveDataSource.getJdbcUrl());
}
diff --git
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
index 5a16194..8f21d5e 100644
---
a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
+++
b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/worker/task/sql/SqlTask.java
@@ -244,6 +244,9 @@ public class SqlTask extends AbstractTask {
PreparedStatement stmt = null;
ResultSet resultSet = null;
try {
+
+ baseDataSource.setConnParams(sqlParameters.getConnParams());
+
// create connection
connection = baseDataSource.getConnection();
// create temp function
diff --git a/pom.xml b/pom.xml
index a07fde2..2829dca 100644
--- a/pom.xml
+++ b/pom.xml
@@ -990,6 +990,7 @@
<include>**/dao/AlertDaoTest.java</include>
<include>**/dao/datasource/OracleDataSourceTest.java</include>
<include>**/dao/datasource/HiveDataSourceTest.java</include>
+
<include>**/dao/datasource/BaseDataSourceTest.java</include>
<include>**/dao/upgrade/ProcessDefinitionDaoTest.java</include>
<include>**/dao/upgrade/WokrerGrouopDaoTest.java</include>
<include>**/dao/upgrade/UpgradeDaoTest.java</include>
