This is an automated email from the ASF dual-hosted git repository. ngangam pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
commit 013a693649c82ee995f750910a360d8e6de4c0f7 Author: Naveen Gangam <ngan...@cloudera.com> AuthorDate: Tue Nov 17 23:46:50 2020 -0500 Adding a qtest and fixing type for default db --- ql/src/test/queries/clientpositive/dataconnector.q | 71 +++++++ .../results/clientpositive/dataconnector.q.out | 205 +++++++++++++++++++++ .../hadoop/hive/metastore/HiveMetaStore.java | 2 +- .../apache/hadoop/hive/metastore/ObjectStore.java | 4 +- .../DataConnectorProviderFactory.java | 7 +- 5 files changed, 283 insertions(+), 6 deletions(-) diff --git a/ql/src/test/queries/clientpositive/dataconnector.q b/ql/src/test/queries/clientpositive/dataconnector.q new file mode 100644 index 0000000..c21524a --- /dev/null +++ b/ql/src/test/queries/clientpositive/dataconnector.q @@ -0,0 +1,71 @@ +-- SORT_QUERY_RESULTS +SHOW CONNECTORS; + +-- CREATE with comment +CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1"); +SHOW CONNECTORS; + +-- CREATE INE already exists +CREATE CONNECTOR IF NOT EXISTS mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1"); +SHOW CONNECTORS; + +-- CREATE INE already exists +CREATE CONNECTOR IF NOT EXISTS derby_test +TYPE 'derby' +URL 'jdbc:derby:./target/tmp/junit_metastore_db;create=true' +COMMENT 'test derby connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="APP", +"hive.sql.dbcp.password"="mine"); + +-- DROP +DROP CONNECTOR mysql_test; +SHOW CONNECTORS; + +-- DROP IE exists +DROP CONNECTOR IF EXISTS mysql_test; +SHOW CONNECTORS; + +-- recreate with same name +CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1"); +SHOW CONNECTORS; + +CREATE REMOTE DATABASE db_derby USING derby_test with DBPROPERTIES("connector.remoteDbName"="APP"); +SHOW DATABASES; +USE db_derby; +SHOW TABLES; + +-- alter connector set URL +alter connector mysql_test set URL 'jdbc:mysql://nightly1.apache.org:3306/hive2'; +DESCRIBE CONNECTOR extended mysql_test; + +-- alter connector set DCPROPERTIES +alter connector mysql_test set DCPROPERTIES("hive.sql.dbcp.username"="hive2","hive.sql.dbcp.password"="hive2"); +DESCRIBE CONNECTOR extended mysql_test; + +-- alter connector set owner +alter connector mysql_test set OWNER USER newuser; +DESCRIBE CONNECTOR extended mysql_test; + +DROP DATABASE db_derby; +SHOW DATABASES; +DROP CONNECTOR mysql_test; +SHOW CONNECTORS; diff --git a/ql/src/test/results/clientpositive/dataconnector.q.out b/ql/src/test/results/clientpositive/dataconnector.q.out new file mode 100644 index 0000000..8b678c2 --- /dev/null +++ b/ql/src/test/results/clientpositive/dataconnector.q.out @@ -0,0 +1,205 @@ +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +PREHOOK: query: CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +PREHOOK: type: CREATEDATACONNECTOR +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +POSTHOOK: type: CREATEDATACONNECTOR +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +mysql_test +PREHOOK: query: CREATE CONNECTOR IF NOT EXISTS mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +PREHOOK: type: CREATEDATACONNECTOR +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: CREATE CONNECTOR IF NOT EXISTS mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +POSTHOOK: type: CREATEDATACONNECTOR +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +mysql_test +PREHOOK: query: CREATE CONNECTOR IF NOT EXISTS derby_test +TYPE 'derby' +#### A masked pattern was here #### +COMMENT 'test derby connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="APP", +"hive.sql.dbcp.password"="mine") +PREHOOK: type: CREATEDATACONNECTOR +PREHOOK: Output: connector:derby_test +POSTHOOK: query: CREATE CONNECTOR IF NOT EXISTS derby_test +TYPE 'derby' +#### A masked pattern was here #### +COMMENT 'test derby connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="APP", +"hive.sql.dbcp.password"="mine") +POSTHOOK: type: CREATEDATACONNECTOR +POSTHOOK: Output: connector:derby_test +PREHOOK: query: DROP CONNECTOR mysql_test +PREHOOK: type: DROPDATACONNECTOR +PREHOOK: Input: connector:mysql_test +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: DROP CONNECTOR mysql_test +POSTHOOK: type: DROPDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +derby_test +PREHOOK: query: DROP CONNECTOR IF EXISTS mysql_test +PREHOOK: type: DROPDATACONNECTOR +POSTHOOK: query: DROP CONNECTOR IF EXISTS mysql_test +POSTHOOK: type: DROPDATACONNECTOR +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +derby_test +PREHOOK: query: CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +PREHOOK: type: CREATEDATACONNECTOR +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +POSTHOOK: type: CREATEDATACONNECTOR +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +derby_test +mysql_test +PREHOOK: query: CREATE REMOTE DATABASE db_derby USING derby_test with DBPROPERTIES("connector.remoteDbName"="APP") +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:db_derby +#### A masked pattern was here #### +POSTHOOK: query: CREATE REMOTE DATABASE db_derby USING derby_test with DBPROPERTIES("connector.remoteDbName"="APP") +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:db_derby +#### A masked pattern was here #### +PREHOOK: query: SHOW DATABASES +PREHOOK: type: SHOWDATABASES +POSTHOOK: query: SHOW DATABASES +POSTHOOK: type: SHOWDATABASES +db_derby +default +PREHOOK: query: USE db_derby +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:db_derby +POSTHOOK: query: USE db_derby +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:db_derby +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:db_derby +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:db_derby +PREHOOK: query: alter connector mysql_test set URL 'jdbc:mysql://nightly1.apache.org:3306/hive2' +PREHOOK: type: ALTERDATACONNECTOR_ +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: alter connector mysql_test set URL 'jdbc:mysql://nightly1.apache.org:3306/hive2' +POSTHOOK: type: ALTERDATACONNECTOR_ +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: DESCRIBE CONNECTOR extended mysql_test +PREHOOK: type: DESCDATACONNECTOR +PREHOOK: Input: connector:mysql_test +POSTHOOK: query: DESCRIBE CONNECTOR extended mysql_test +POSTHOOK: type: DESCDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +mysql_test mysql jdbc:mysql://nightly1.apache.org:3306/hive2 hive_test_user USER test connector {hive.sql.dbcp.password=hive1, hive.sql.dbcp.username=hive1} +PREHOOK: query: alter connector mysql_test set DCPROPERTIES("hive.sql.dbcp.username"="hive2","hive.sql.dbcp.password"="hive2") +PREHOOK: type: ALTERDATACONNECTOR +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: alter connector mysql_test set DCPROPERTIES("hive.sql.dbcp.username"="hive2","hive.sql.dbcp.password"="hive2") +POSTHOOK: type: ALTERDATACONNECTOR +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: DESCRIBE CONNECTOR extended mysql_test +PREHOOK: type: DESCDATACONNECTOR +PREHOOK: Input: connector:mysql_test +POSTHOOK: query: DESCRIBE CONNECTOR extended mysql_test +POSTHOOK: type: DESCDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +mysql_test mysql jdbc:mysql://nightly1.apache.org:3306/hive2 hive_test_user USER test connector {hive.sql.dbcp.password=hive2, hive.sql.dbcp.username=hive2} +PREHOOK: query: alter connector mysql_test set OWNER USER newuser +PREHOOK: type: ALTERDATABASE_OWNER +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: alter connector mysql_test set OWNER USER newuser +POSTHOOK: type: ALTERDATABASE_OWNER +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: DESCRIBE CONNECTOR extended mysql_test +PREHOOK: type: DESCDATACONNECTOR +PREHOOK: Input: connector:mysql_test +POSTHOOK: query: DESCRIBE CONNECTOR extended mysql_test +POSTHOOK: type: DESCDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +mysql_test mysql jdbc:mysql://nightly1.apache.org:3306/hive2 newuser USER test connector {hive.sql.dbcp.password=hive2, hive.sql.dbcp.username=hive2} +PREHOOK: query: DROP DATABASE db_derby +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:db_derby +PREHOOK: Output: database:db_derby +POSTHOOK: query: DROP DATABASE db_derby +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:db_derby +POSTHOOK: Output: database:db_derby +PREHOOK: query: SHOW DATABASES +PREHOOK: type: SHOWDATABASES +POSTHOOK: query: SHOW DATABASES +POSTHOOK: type: SHOWDATABASES +default +PREHOOK: query: DROP CONNECTOR mysql_test +PREHOOK: type: DROPDATACONNECTOR +PREHOOK: Input: connector:mysql_test +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: DROP CONNECTOR mysql_test +POSTHOOK: type: DROPDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +derby_test diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 42c7d6f..ec0fa00 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -5998,7 +5998,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { String[] parsedDbName = parseDbName(dbname, conf); try { db = get_database_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME]); - if (db != null) { + if (db != null && db.getType() != null) { if (db.getType().equals(DatabaseType.REMOTE)) { return DataConnectorProviderFactory.getDataConnectorProvider(db).getTableNames(); } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 63a3df5..7ade2fc 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -785,10 +785,10 @@ public class ObjectStore implements RawStore, Configurable { mdb.setOwnerName(db.getOwnerName()); mdb.setDataConnectorName(db.getConnector_name()); mdb.setRemoteDatabaseName(db.getRemote_dbname()); - if (db.getType() == DatabaseType.NATIVE) { + if (db.getType() == null) { mdb.setType("NATIVE"); } else { - mdb.setType("REMOTE"); + mdb.setType(db.getType().toString()); } PrincipalType ownerType = db.getOwnerType(); mdb.setOwnerType((null == ownerType ? PrincipalType.USER.name() : ownerType.name())); diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/DataConnectorProviderFactory.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/DataConnectorProviderFactory.java index 7f321a8..bb2d972 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/DataConnectorProviderFactory.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/dataconnector/DataConnectorProviderFactory.java @@ -64,13 +64,12 @@ public class DataConnectorProviderFactory { case POSTGRES_TYPE: try { provider = JDBCConnectorProviderFactory.get(scopedDb, connector); - cache.put(db.getConnector_name(), provider); } catch (Exception e) { throw new MetaException("Could not instantiate a provider for database " + db.getName()); } - ; + break; default: - ; + throw new MetaException("Data connector of type " + connector.getType() + " not implemented yet"); } cache.put(connector.getName().toLowerCase(), provider); return provider; @@ -94,6 +93,8 @@ public class DataConnectorProviderFactory { provider.close(); } catch(Exception e) { LOG.warn("Exception invoking close on dataconnectorprovider:" + provider, e); + } finally { + cache.clear(); } } }