This is an automated email from the ASF dual-hosted git repository. ngangam pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
commit df8cb1146163acf3166382739e18522c3df2448a Author: Naveen Gangam <ngan...@cloudera.com> AuthorDate: Mon Nov 23 15:54:56 2020 -0500 HIVE-24396: Follow up test failure fixes --- .../hadoop/hive/ql/parse/IdentifiersParser.g | 2 +- .../authorization/plugin/HiveOperationType.java | 7 + .../llap/alter_change_db_location.q.out | 3 +- .../clientpositive/llap/database_location.q.out | 47 ++--- .../clientpositive/llap/dataconnector.q.out | 205 +++++++++++++++++++++ .../clientpositive/llap/db_ddl_explain.q.out | 2 +- .../clientpositive/llap/unicode_comments.q.out | 2 +- 7 files changed, 236 insertions(+), 32 deletions(-) diff --git a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g index d836b80..cbd45d3 100644 --- a/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g +++ b/parser/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g @@ -927,7 +927,7 @@ nonReserved | KW_SHOW | KW_SHOW_DATABASE | KW_SKEWED | KW_SORT | KW_SORTED | KW_SSL | KW_STATISTICS | KW_STORED | KW_AST | KW_STREAMTABLE | KW_STRING | KW_STRUCT | KW_TABLES | KW_TBLPROPERTIES | KW_TEMPORARY | KW_TERMINATED | KW_TINYINT | KW_TOUCH | KW_TRANSACTIONAL | KW_TRANSACTIONS | KW_TYPE | KW_UNARCHIVE | KW_UNDO | KW_UNIONTYPE | KW_UNLOCK | KW_UNSET - | KW_UNSIGNED | KW_URI | KW_USE | KW_UTC | KW_UTCTIMESTAMP | KW_VALUE_TYPE | KW_VIEW | KW_WEEK | KW_WHILE | KW_YEAR + | KW_UNSIGNED | KW_URI | KW_URL | KW_USE | KW_UTC | KW_UTCTIMESTAMP | KW_VALUE_TYPE | KW_VIEW | KW_WEEK | KW_WHILE | KW_YEAR | KW_WORK | KW_TRANSACTION | KW_WRITE diff --git a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java index 2511e0b..0d88199 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java @@ -34,7 +34,9 @@ public enum HiveOperationType { REPLLOAD, REPLSTATUS, CREATEDATABASE, + CREATEDATACONNECTOR, DROPDATABASE, + DROPDATACONNECTOR, SWITCHDATABASE, LOCKDB, UNLOCKDB, @@ -71,6 +73,7 @@ public enum HiveOperationType { ALTERTABLE_UPDATETABLESTATS, ALTERTABLE_UPDATEPARTSTATS, SHOWDATABASES, + SHOWDATACONNECTORS, SHOWTABLES, SHOWCOLUMNS, SHOW_TABLESTATUS, @@ -118,7 +121,11 @@ public enum HiveOperationType { ALTERDATABASE, ALTERDATABASE_OWNER, ALTERDATABASE_LOCATION, + ALTERDATACONNECTOR, + ALTERDATACONNECTOR_OWNER, + ALTERDATACONNECTOR_URL, DESCDATABASE, + DESCDATACONNECTOR, ALTERTABLE_MERGEFILES, ALTERPARTITION_MERGEFILES, ALTERTABLE_SKEWED, diff --git a/ql/src/test/results/clientpositive/llap/alter_change_db_location.q.out b/ql/src/test/results/clientpositive/llap/alter_change_db_location.q.out index 2469cea..d07eaa6 100644 --- a/ql/src/test/results/clientpositive/llap/alter_change_db_location.q.out +++ b/ql/src/test/results/clientpositive/llap/alter_change_db_location.q.out @@ -1,11 +1,10 @@ #### A masked pattern was here #### PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:newDB -PREHOOK: Output: hdfs://### HDFS PATH ### #### A masked pattern was here #### POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:newDB -POSTHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### PREHOOK: query: describe database extended newDB PREHOOK: type: DESCDATABASE PREHOOK: Input: database:newdb diff --git a/ql/src/test/results/clientpositive/llap/database_location.q.out b/ql/src/test/results/clientpositive/llap/database_location.q.out index b998e7e..1cb5127 100644 --- a/ql/src/test/results/clientpositive/llap/database_location.q.out +++ b/ql/src/test/results/clientpositive/llap/database_location.q.out @@ -72,13 +72,13 @@ COMMENT 'database 2' #### A masked pattern was here #### PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:db2 -PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### POSTHOOK: query: CREATE DATABASE db2 COMMENT 'database 2' #### A masked pattern was here #### POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:db2 -POSTHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### PREHOOK: query: DESCRIBE DATABASE EXTENDED db2 PREHOOK: type: DESCDATABASE PREHOOK: Input: database:db2 @@ -147,15 +147,13 @@ CREATE DATABASE db3 #### A masked pattern was here #### PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:db3 -PREHOOK: Output: hdfs://### HDFS PATH ### -PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### POSTHOOK: query: EXPLAIN CREATE DATABASE db3 #### A masked pattern was here #### POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:db3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### STAGE DEPENDENCIES: Stage-0 is a root stage @@ -163,22 +161,19 @@ STAGE PLANS: Stage: Stage-0 Create Database database type: - locationUri: hdfs://### HDFS PATH ### - managed location uri: hdfs://### HDFS PATH ### +#### A masked pattern was here #### name: db3 PREHOOK: query: CREATE DATABASE db3 #### A masked pattern was here #### PREHOOK: type: CREATEDATABASE PREHOOK: Output: database:db3 -PREHOOK: Output: hdfs://### HDFS PATH ### -PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### POSTHOOK: query: CREATE DATABASE db3 #### A masked pattern was here #### POSTHOOK: type: CREATEDATABASE POSTHOOK: Output: database:db3 -POSTHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### PREHOOK: query: SHOW CREATE DATABASE db3 PREHOOK: type: SHOW_CREATEDATABASE PREHOOK: Input: database:db3 @@ -187,26 +182,26 @@ POSTHOOK: type: SHOW_CREATEDATABASE POSTHOOK: Input: database:db3 CREATE DATABASE `db3` LOCATION - 'hdfs://### HDFS PATH ###' +#### A masked pattern was here #### MANAGEDLOCATION - 'hdfs://### HDFS PATH ###' +#### A masked pattern was here #### PREHOOK: query: DESCRIBE DATABASE db3 PREHOOK: type: DESCDATABASE PREHOOK: Input: database:db3 POSTHOOK: query: DESCRIBE DATABASE db3 POSTHOOK: type: DESCDATABASE POSTHOOK: Input: database:db3 -db3 location/in/test hdfs://### HDFS PATH ### hive_test_user USER +#### A masked pattern was here #### PREHOOK: query: EXPLAIN #### A masked pattern was here #### PREHOOK: type: ALTERDATABASE_LOCATION PREHOOK: Output: database:db3 -PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### POSTHOOK: query: EXPLAIN #### A masked pattern was here #### POSTHOOK: type: ALTERDATABASE_LOCATION POSTHOOK: Output: database:db3 -POSTHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### STAGE DEPENDENCIES: Stage-0 is a root stage @@ -214,33 +209,32 @@ STAGE PLANS: Stage: Stage-0 #### A masked pattern was here #### name: db3 - location: hdfs://### HDFS PATH ### +#### A masked pattern was here #### #### A masked pattern was here #### PREHOOK: type: ALTERDATABASE_LOCATION PREHOOK: Output: database:db3 -PREHOOK: Output: hdfs://### HDFS PATH ### #### A masked pattern was here #### POSTHOOK: type: ALTERDATABASE_LOCATION POSTHOOK: Output: database:db3 -POSTHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### PREHOOK: query: DESCRIBE DATABASE db3 PREHOOK: type: DESCDATABASE PREHOOK: Input: database:db3 POSTHOOK: query: DESCRIBE DATABASE db3 POSTHOOK: type: DESCDATABASE POSTHOOK: Input: database:db3 -db3 location/in/test hdfs://### HDFS PATH ### hive_test_user USER +#### A masked pattern was here #### PREHOOK: query: EXPLAIN #### A masked pattern was here #### PREHOOK: type: ALTERDATABASE_LOCATION PREHOOK: Output: database:db3 -PREHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### POSTHOOK: query: EXPLAIN #### A masked pattern was here #### POSTHOOK: type: ALTERDATABASE_LOCATION POSTHOOK: Output: database:db3 -POSTHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### STAGE DEPENDENCIES: Stage-0 is a root stage @@ -248,20 +242,19 @@ STAGE PLANS: Stage: Stage-0 #### A masked pattern was here #### name: db3 - managed location: hdfs://### HDFS PATH ### +#### A masked pattern was here #### #### A masked pattern was here #### PREHOOK: type: ALTERDATABASE_LOCATION PREHOOK: Output: database:db3 -PREHOOK: Output: hdfs://### HDFS PATH ### #### A masked pattern was here #### POSTHOOK: type: ALTERDATABASE_LOCATION POSTHOOK: Output: database:db3 -POSTHOOK: Output: hdfs://### HDFS PATH ### +#### A masked pattern was here #### PREHOOK: query: DESCRIBE DATABASE db3 PREHOOK: type: DESCDATABASE PREHOOK: Input: database:db3 POSTHOOK: query: DESCRIBE DATABASE db3 POSTHOOK: type: DESCDATABASE POSTHOOK: Input: database:db3 -db3 location/in/test hdfs://### HDFS PATH ### hive_test_user USER +#### A masked pattern was here #### diff --git a/ql/src/test/results/clientpositive/llap/dataconnector.q.out b/ql/src/test/results/clientpositive/llap/dataconnector.q.out new file mode 100644 index 0000000..8b678c2 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/dataconnector.q.out @@ -0,0 +1,205 @@ +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +PREHOOK: query: CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +PREHOOK: type: CREATEDATACONNECTOR +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +POSTHOOK: type: CREATEDATACONNECTOR +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +mysql_test +PREHOOK: query: CREATE CONNECTOR IF NOT EXISTS mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +PREHOOK: type: CREATEDATACONNECTOR +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: CREATE CONNECTOR IF NOT EXISTS mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +POSTHOOK: type: CREATEDATACONNECTOR +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +mysql_test +PREHOOK: query: CREATE CONNECTOR IF NOT EXISTS derby_test +TYPE 'derby' +#### A masked pattern was here #### +COMMENT 'test derby connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="APP", +"hive.sql.dbcp.password"="mine") +PREHOOK: type: CREATEDATACONNECTOR +PREHOOK: Output: connector:derby_test +POSTHOOK: query: CREATE CONNECTOR IF NOT EXISTS derby_test +TYPE 'derby' +#### A masked pattern was here #### +COMMENT 'test derby connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="APP", +"hive.sql.dbcp.password"="mine") +POSTHOOK: type: CREATEDATACONNECTOR +POSTHOOK: Output: connector:derby_test +PREHOOK: query: DROP CONNECTOR mysql_test +PREHOOK: type: DROPDATACONNECTOR +PREHOOK: Input: connector:mysql_test +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: DROP CONNECTOR mysql_test +POSTHOOK: type: DROPDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +derby_test +PREHOOK: query: DROP CONNECTOR IF EXISTS mysql_test +PREHOOK: type: DROPDATACONNECTOR +POSTHOOK: query: DROP CONNECTOR IF EXISTS mysql_test +POSTHOOK: type: DROPDATACONNECTOR +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +derby_test +PREHOOK: query: CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +PREHOOK: type: CREATEDATACONNECTOR +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: CREATE CONNECTOR mysql_test +TYPE 'mysql' +URL 'jdbc:mysql://nightly1.apache.org:3306/hive1' +COMMENT 'test connector' +WITH DCPROPERTIES ( +"hive.sql.dbcp.username"="hive1", +"hive.sql.dbcp.password"="hive1") +POSTHOOK: type: CREATEDATACONNECTOR +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +derby_test +mysql_test +PREHOOK: query: CREATE REMOTE DATABASE db_derby USING derby_test with DBPROPERTIES("connector.remoteDbName"="APP") +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:db_derby +#### A masked pattern was here #### +POSTHOOK: query: CREATE REMOTE DATABASE db_derby USING derby_test with DBPROPERTIES("connector.remoteDbName"="APP") +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:db_derby +#### A masked pattern was here #### +PREHOOK: query: SHOW DATABASES +PREHOOK: type: SHOWDATABASES +POSTHOOK: query: SHOW DATABASES +POSTHOOK: type: SHOWDATABASES +db_derby +default +PREHOOK: query: USE db_derby +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:db_derby +POSTHOOK: query: USE db_derby +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:db_derby +PREHOOK: query: SHOW TABLES +PREHOOK: type: SHOWTABLES +PREHOOK: Input: database:db_derby +POSTHOOK: query: SHOW TABLES +POSTHOOK: type: SHOWTABLES +POSTHOOK: Input: database:db_derby +PREHOOK: query: alter connector mysql_test set URL 'jdbc:mysql://nightly1.apache.org:3306/hive2' +PREHOOK: type: ALTERDATACONNECTOR_ +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: alter connector mysql_test set URL 'jdbc:mysql://nightly1.apache.org:3306/hive2' +POSTHOOK: type: ALTERDATACONNECTOR_ +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: DESCRIBE CONNECTOR extended mysql_test +PREHOOK: type: DESCDATACONNECTOR +PREHOOK: Input: connector:mysql_test +POSTHOOK: query: DESCRIBE CONNECTOR extended mysql_test +POSTHOOK: type: DESCDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +mysql_test mysql jdbc:mysql://nightly1.apache.org:3306/hive2 hive_test_user USER test connector {hive.sql.dbcp.password=hive1, hive.sql.dbcp.username=hive1} +PREHOOK: query: alter connector mysql_test set DCPROPERTIES("hive.sql.dbcp.username"="hive2","hive.sql.dbcp.password"="hive2") +PREHOOK: type: ALTERDATACONNECTOR +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: alter connector mysql_test set DCPROPERTIES("hive.sql.dbcp.username"="hive2","hive.sql.dbcp.password"="hive2") +POSTHOOK: type: ALTERDATACONNECTOR +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: DESCRIBE CONNECTOR extended mysql_test +PREHOOK: type: DESCDATACONNECTOR +PREHOOK: Input: connector:mysql_test +POSTHOOK: query: DESCRIBE CONNECTOR extended mysql_test +POSTHOOK: type: DESCDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +mysql_test mysql jdbc:mysql://nightly1.apache.org:3306/hive2 hive_test_user USER test connector {hive.sql.dbcp.password=hive2, hive.sql.dbcp.username=hive2} +PREHOOK: query: alter connector mysql_test set OWNER USER newuser +PREHOOK: type: ALTERDATABASE_OWNER +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: alter connector mysql_test set OWNER USER newuser +POSTHOOK: type: ALTERDATABASE_OWNER +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: DESCRIBE CONNECTOR extended mysql_test +PREHOOK: type: DESCDATACONNECTOR +PREHOOK: Input: connector:mysql_test +POSTHOOK: query: DESCRIBE CONNECTOR extended mysql_test +POSTHOOK: type: DESCDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +mysql_test mysql jdbc:mysql://nightly1.apache.org:3306/hive2 newuser USER test connector {hive.sql.dbcp.password=hive2, hive.sql.dbcp.username=hive2} +PREHOOK: query: DROP DATABASE db_derby +PREHOOK: type: DROPDATABASE +PREHOOK: Input: database:db_derby +PREHOOK: Output: database:db_derby +POSTHOOK: query: DROP DATABASE db_derby +POSTHOOK: type: DROPDATABASE +POSTHOOK: Input: database:db_derby +POSTHOOK: Output: database:db_derby +PREHOOK: query: SHOW DATABASES +PREHOOK: type: SHOWDATABASES +POSTHOOK: query: SHOW DATABASES +POSTHOOK: type: SHOWDATABASES +default +PREHOOK: query: DROP CONNECTOR mysql_test +PREHOOK: type: DROPDATACONNECTOR +PREHOOK: Input: connector:mysql_test +PREHOOK: Output: connector:mysql_test +POSTHOOK: query: DROP CONNECTOR mysql_test +POSTHOOK: type: DROPDATACONNECTOR +POSTHOOK: Input: connector:mysql_test +POSTHOOK: Output: connector:mysql_test +PREHOOK: query: SHOW CONNECTORS +PREHOOK: type: SHOWDATACONNECTORS +POSTHOOK: query: SHOW CONNECTORS +POSTHOOK: type: SHOWDATACONNECTORS +derby_test diff --git a/ql/src/test/results/clientpositive/llap/db_ddl_explain.q.out b/ql/src/test/results/clientpositive/llap/db_ddl_explain.q.out index 3ba6778..257d697 100644 --- a/ql/src/test/results/clientpositive/llap/db_ddl_explain.q.out +++ b/ql/src/test/results/clientpositive/llap/db_ddl_explain.q.out @@ -122,7 +122,7 @@ POSTHOOK: type: SHOW_CREATEDATABASE POSTHOOK: Input: database:d CREATE DATABASE `d` LOCATION - 'hdfs://### HDFS PATH ###' +#### A masked pattern was here #### WITH DBPROPERTIES ( 'test'='yesthisis') PREHOOK: query: EXPLAIN USE d diff --git a/ql/src/test/results/clientpositive/llap/unicode_comments.q.out b/ql/src/test/results/clientpositive/llap/unicode_comments.q.out index 30b2fb9..022370b 100644 --- a/ql/src/test/results/clientpositive/llap/unicode_comments.q.out +++ b/ql/src/test/results/clientpositive/llap/unicode_comments.q.out @@ -60,7 +60,7 @@ STORED AS INPUTFORMAT OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' LOCATION - 'hdfs://### HDFS PATH ###' +#### A masked pattern was here #### TBLPROPERTIES ( 'bucketing_version'='2', #### A masked pattern was here ####