This is an automated email from the ASF dual-hosted git repository. peacewong pushed a commit to branch dev-1.2.0 in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
commit 92a25c07eb533b8e9042266546af647b02af9b3f Author: CCweixiao <[email protected]> AuthorDate: Fri May 27 19:16:48 2022 +0800 [feture-2140] Unify database connection parameters in JDBC engine and extract variables. --- .../engineplugin/jdbc/ConnectionManager.java | 31 ++++---- .../jdbc/constant/JDBCEngineConnConstant.java | 29 ++++++++ .../jdbc/executer/JDBCEngineConnExecutor.scala | 20 ++--- .../engineplugin/jdbc/ConnectionManagerTest.java | 86 ++++++++++++++++++++++ .../engineplugin/jdbc/TestConnectionManager.java | 82 --------------------- 5 files changed, 141 insertions(+), 107 deletions(-) diff --git a/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java index 9d9ae3bc1..00f2678db 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java +++ b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManager.java @@ -26,6 +26,7 @@ import org.apache.commons.lang.StringUtils; import javax.sql.DataSource; +import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,10 +101,10 @@ public class ConnectionManager { private void validateURL(String url) { if (StringUtils.isEmpty(url)) { - throw new NullPointerException("jdbc.url cannot be null."); + throw new NullPointerException(JDBCEngineConnConstant.JDBC_URL + " cannot be null."); } if (!url.matches("jdbc:\\w+://\\S+:[0-9]{2,6}(/\\S*)?")) { - throw new IllegalArgumentException("Unknown jdbc.url " + url); + throw new IllegalArgumentException("Unknown the jdbc url: " + url); } for (String supportedDBName : supportedDBNames) { if (url.indexOf(supportedDBName) > 0) { @@ -124,8 +125,8 @@ public class ConnectionManager { protected DataSource createDataSources(Map<String, String> properties) throws SQLException { String url = getJdbcUrl(properties); - String username = properties.getOrDefault("jdbc.username", ""); - String password = StringUtils.trim(properties.getOrDefault("jdbc.password", "")); + String username = properties.getOrDefault(JDBCEngineConnConstant.JDBC_USERNAME, ""); + String password = StringUtils.trim(properties.getOrDefault(JDBCEngineConnConstant.JDBC_PASSWORD, "")); int index = url.indexOf(":") + 1; String dbType = url.substring(index, url.indexOf(":", index)); Properties props = new Properties(); @@ -140,9 +141,9 @@ public class ConnectionManager { props.put("validationQuery", this.supportedDBsValidQuery.get(dbType)); if (isKerberosAuthType(properties)) { - String jdbcProxyUser = properties.get("jdbc.proxy.user"); + String jdbcProxyUser = properties.get(JDBCEngineConnConstant.JDBC_PROXY_USER); // need proxy user - String proxyUserProperty = properties.get("jdbc.proxy.user.property"); + String proxyUserProperty = properties.get(JDBCEngineConnConstant.JDBC_PROXY_USER_PROPERTY); if (StringUtils.isNotBlank(proxyUserProperty)) { url = url.concat(";").concat(proxyUserProperty + "=" + jdbcProxyUser); props.put("url", url); @@ -185,17 +186,17 @@ public class ConnectionManager { connection = getConnection(url, properties); break; case KERBEROS: - final String keytab = properties.get("jdbc.keytab.location"); - final String principal = properties.get("jdbc.principal"); + final String keytab = properties.get(JDBCEngineConnConstant.JDBC_KERBEROS_AUTH_TYPE_KEYTAB_LOCATION); + final String principal = properties.get(JDBCEngineConnConstant.JDBC_KERBEROS_AUTH_TYPE_PRINCIPAL); KerberosUtils.createKerberosSecureConfiguration(keytab, principal); connection = getConnection(url, properties); break; case USERNAME: - if (StringUtils.isEmpty(properties.get("jdbc.username"))) { - throw new SQLException("jdbc.username is not empty."); + if (StringUtils.isEmpty(properties.get(JDBCEngineConnConstant.JDBC_USERNAME))) { + throw new SQLException(JDBCEngineConnConstant.JDBC_USERNAME + " is not empty."); } - if (StringUtils.isEmpty(properties.get("jdbc.password"))) { - throw new SQLException("jdbc.password is not empty."); + if (StringUtils.isEmpty(properties.get(JDBCEngineConnConstant.JDBC_PASSWORD))) { + throw new SQLException(JDBCEngineConnConstant.JDBC_PASSWORD + " is not empty."); } connection = getConnection(url, properties); break; @@ -231,9 +232,9 @@ public class ConnectionManager { } private String getJdbcUrl(Map<String, String> properties) throws SQLException { - String url = properties.get("jdbc.url"); + String url = properties.get(JDBCEngineConnConstant.JDBC_URL); if (StringUtils.isEmpty(url)) { - throw new SQLException("jdbc.url is not empty."); + throw new SQLException(JDBCEngineConnConstant.JDBC_URL + " is not empty."); } url = clearUrl(url); validateURL(url); @@ -249,7 +250,7 @@ public class ConnectionManager { } private JdbcAuthType getJdbcAuthType(Map<String, String> properties) { - String authType = properties.getOrDefault("jdbc.auth.type", USERNAME.getAuthType()); + String authType = properties.getOrDefault(JDBCEngineConnConstant.JDBC_AUTH_TYPE, USERNAME.getAuthType()); if (authType == null || authType.trim().length() == 0) return of(USERNAME.getAuthType()); return of(authType.trim().toUpperCase()); } diff --git a/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/constant/JDBCEngineConnConstant.java b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/constant/JDBCEngineConnConstant.java new file mode 100644 index 000000000..54fa76fc4 --- /dev/null +++ b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/java/org/apache/linkis/manager/engineplugin/jdbc/constant/JDBCEngineConnConstant.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.jdbc.constant; + +public class JDBCEngineConnConstant { + public static final String JDBC_URL = "wds.linkis.jdbc.connect.url"; + public static final String JDBC_USERNAME = "wds.linkis.jdbc.username"; + public static final String JDBC_PASSWORD = "wds.linkis.jdbc.password"; + public static final String JDBC_AUTH_TYPE = "wds.linkis.jdbc.auth.type"; + public static final String JDBC_KERBEROS_AUTH_TYPE_PRINCIPAL = "wds.linkis.jdbc.principal"; + public static final String JDBC_KERBEROS_AUTH_TYPE_KEYTAB_LOCATION = "wds.linkis.jdbc.keytab.location"; + public static final String JDBC_PROXY_USER_PROPERTY = "wds.linkis.jdbc.proxy.user.property"; + public static final String JDBC_PROXY_USER = "wds.linkis.jdbc.proxy.user"; +} diff --git a/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executer/JDBCEngineConnExecutor.scala b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executer/JDBCEngineConnExecutor.scala index 1c8f2cbac..51ad30a61 100644 --- a/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executer/JDBCEngineConnExecutor.scala +++ b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/main/scala/org/apache/linkis/manager/engineplugin/jdbc/executer/JDBCEngineConnExecutor.scala @@ -19,7 +19,6 @@ package org.apache.linkis.manager.engineplugin.jdbc.executer import java.sql.{Connection, Statement} import java.util - import org.apache.linkis.common.utils.{OverloadUtils, Utils} import org.apache.linkis.engineconn.computation.executor.execute.{ConcurrentComputationExecutor, EngineExecutionContext} import org.apache.linkis.engineconn.core.EngineConnObject @@ -41,6 +40,7 @@ import org.apache.linkis.manager.label.entity.engine.{EngineTypeLabel, UserCreat import org.apache.linkis.protocol.CacheableProtocol import org.springframework.util.CollectionUtils import org.apache.linkis.governance.common.paser.SQLCodeParser +import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant import scala.collection.JavaConversions._ import scala.collection.mutable.ArrayBuffer @@ -67,17 +67,17 @@ class JDBCEngineConnExecutor(override val outputPrintLimit: Int, val id: Int) ex val realCode = code.trim() val properties = engineExecutorContext.getProperties.asInstanceOf[util.Map[String, String]] - if (properties.get("jdbc.url") == null) { + if (properties.get(JDBCEngineConnConstant.JDBC_URL) == null) { info(s"jdbc url is empty, adding now...") val globalConfig = Utils.tryAndWarn(JDBCEngineConfig.getCacheMap(engineExecutorContext.getLabels)) - properties.put("jdbc.url", globalConfig.get("wds.linkis.jdbc.connect.url")) - properties.put("jdbc.username", globalConfig.get("wds.linkis.jdbc.username")) - properties.put("jdbc.password", globalConfig.get("wds.linkis.jdbc.password")) - properties.put("jdbc.auth.type", globalConfig.get("wds.linkis.jdbc.auth.type")) - properties.put("jdbc.principal", globalConfig.get("wds.linkis.jdbc.principal")) - properties.put("jdbc.keytab.location", globalConfig.get("wds.linkis.jdbc.keytab.location")) - properties.put("jdbc.proxy.user.property", globalConfig.getOrDefault("wds.linkis.jdbc.proxy.user.property", "")) - properties.put("jdbc.proxy.user", globalConfig.getOrDefault("wds.linkis.jdbc.proxy.user", EngineConnObject.getEngineCreationContext.getUser)) + properties.put(JDBCEngineConnConstant.JDBC_URL, globalConfig.get(JDBCEngineConnConstant.JDBC_URL)) + properties.put(JDBCEngineConnConstant.JDBC_USERNAME, globalConfig.get(JDBCEngineConnConstant.JDBC_USERNAME)) + properties.put(JDBCEngineConnConstant.JDBC_PASSWORD, globalConfig.get(JDBCEngineConnConstant.JDBC_PASSWORD)) + properties.put(JDBCEngineConnConstant.JDBC_AUTH_TYPE, globalConfig.get(JDBCEngineConnConstant.JDBC_AUTH_TYPE)) + properties.put(JDBCEngineConnConstant.JDBC_KERBEROS_AUTH_TYPE_PRINCIPAL, globalConfig.get(JDBCEngineConnConstant.JDBC_KERBEROS_AUTH_TYPE_PRINCIPAL)) + properties.put(JDBCEngineConnConstant.JDBC_KERBEROS_AUTH_TYPE_KEYTAB_LOCATION, globalConfig.get(JDBCEngineConnConstant.JDBC_KERBEROS_AUTH_TYPE_KEYTAB_LOCATION)) + properties.put(JDBCEngineConnConstant.JDBC_PROXY_USER_PROPERTY, globalConfig.getOrDefault(JDBCEngineConnConstant.JDBC_PROXY_USER_PROPERTY, "")) + properties.put(JDBCEngineConnConstant.JDBC_PROXY_USER, globalConfig.getOrDefault(JDBCEngineConnConstant.JDBC_PROXY_USER, EngineConnObject.getEngineCreationContext.getUser)) } info(s"jdbc client begins to run jdbc code:\n ${realCode.trim}") diff --git a/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManagerTest.java b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManagerTest.java new file mode 100644 index 000000000..80dc2860c --- /dev/null +++ b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/ConnectionManagerTest.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.linkis.manager.engineplugin.jdbc; + +import org.apache.commons.dbcp.BasicDataSource; +import org.apache.commons.dbcp.BasicDataSourceFactory; +import org.apache.linkis.manager.engineplugin.jdbc.constant.JDBCEngineConnConstant; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; + +public class ConnectionManagerTest { + @Test + @DisplayName("testCreateJdbcConnAndExecSql") + public void testCreateJdbcConnAndExecSql() throws SQLException { + Map<String, String> properties = new HashMap<>(8); + properties.put(JDBCEngineConnConstant.JDBC_URL, "jdbc:mysql://dev:3306/db?useSSL=false"); + properties.put(JDBCEngineConnConstant.JDBC_USERNAME, "leo"); + properties.put(JDBCEngineConnConstant.JDBC_PASSWORD, "Yyf5211314!"); + properties.put(JDBCEngineConnConstant.JDBC_AUTH_TYPE, "USERNAME"); + properties.put(JDBCEngineConnConstant.JDBC_KERBEROS_AUTH_TYPE_PRINCIPAL, ""); + properties.put(JDBCEngineConnConstant.JDBC_KERBEROS_AUTH_TYPE_KEYTAB_LOCATION, ""); + properties.put(JDBCEngineConnConstant.JDBC_PROXY_USER_PROPERTY, ""); + properties.put(JDBCEngineConnConstant.JDBC_PROXY_USER, ""); + ConnectionManager connectionManager = ConnectionManager.getInstance(); + Connection conn = connectionManager.getConnection(properties); + Statement statement = conn.createStatement(); + ResultSet rs = statement.executeQuery("show databases;"); + while (rs.next()) { + System.out.println(rs.getObject(1)); + } + rs.close(); + statement.close(); + conn.close(); + + } + + @Test + @DisplayName("testExecSql") + public void testExecSql() throws Exception { + Properties properties = new Properties(); + properties.put("driverClassName", "com.mysql.jdbc.Driver"); + properties.put("url", "jdbc:mysql://dev:3306/db?useSSL=false"); + properties.put("username", "leo"); + properties.put("password", "Yyf5211314!"); + properties.put("maxIdle", 20); + properties.put("minIdle", 0); + properties.put("initialSize", 1); + properties.put("testOnBorrow", false); + properties.put("testWhileIdle", true); + properties.put("validationQuery", "select 1"); + BasicDataSource dataSource = (BasicDataSource) BasicDataSourceFactory.createDataSource(properties); + Connection conn = dataSource.getConnection(); + Statement statement = conn.createStatement(); + ResultSet rs = statement.executeQuery("show databases;"); + while (rs.next()) { + System.out.println(rs.getObject(1)); + } + rs.close(); + statement.close(); + conn.close(); + dataSource.close(); + } +} diff --git a/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/TestConnectionManager.java b/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/TestConnectionManager.java deleted file mode 100644 index 2b3b410f1..000000000 --- a/linkis-engineconn-plugins/engineconn-plugins/jdbc/src/test/java/org/apache/linkis/manager/engineplugin/jdbc/TestConnectionManager.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.linkis.manager.engineplugin.jdbc; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.HashMap; -import java.util.Map; - -public class TestConnectionManager { - public static void main(String[] args) throws Exception { - // Pattern pattern = Pattern.compile("^(jdbc:\\w+://\\S+:[0-9]+)\\s*"); - /* String url = "jdbc:mysql://xxx.xxx.xxx.xxx:8504/xx?useUnicode=true&characterEncoding=UTF-8&createDatabaseIfNotExist=true"; - Properties properties = new Properties(); - properties.put("driverClassName", "org.apache.hive.jdbc.HiveDriver"); - properties.put("url", "jdbc:hive2://xxx.xxx.xxx.xxx:10000/"); - properties.put("username", "username"); - properties.put("password", "*****"); - properties.put("maxIdle", 20); - properties.put("minIdle", 0); - properties.put("initialSize", 1); - properties.put("testOnBorrow", false); - properties.put("testWhileIdle", true); - properties.put("validationQuery", "select 1"); - properties.put("initialSize", 1); - BasicDataSource dataSource = (BasicDataSource) BasicDataSourceFactory.createDataSource(properties); - Connection conn = dataSource.getConnection(); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery("show tables"); - while (rs.next()) { - System.out.println(rs.getObject(1)); - } - rs.close(); - statement.close(); - conn.close(); - dataSource.close();*/ - // export LINKIS_JDBC_KERBEROS_REFRESH_INTERVAL=10000 - - Map<String, String> properties = new HashMap<>(8); - properties.put("driverClassName", args[0]); - properties.put("jdbc.url", args[1]); - properties.put("jdbc.username", args[2]); - properties.put("jdbc.password", args[3]); - properties.put("jdbc.auth.type", args[4]); - properties.put("jdbc.principal", args[5]); - properties.put("jdbc.keytab.location", args[6]); - properties.put("jdbc.proxy.user", args[7]); - properties.put("jdbc.proxy.user.property", "hive.server2.proxy.user"); - ConnectionManager connectionManager = ConnectionManager.getInstance(); - connectionManager.startRefreshKerberosLoginStatusThread(); - for (int i = 0; i < 200000; i++) { - Connection conn = connectionManager.getConnection(properties); - Statement statement = conn.createStatement(); - ResultSet rs = statement.executeQuery(args[8]); - while (rs.next()) { - System.out.println(rs.getObject(1)); - } - rs.close(); - statement.close(); - conn.close(); - Thread.sleep(100000); - } - - System.out.println("end ......."); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
