HIVE-19448: Vectorization: sysdb test doesn't work after enabling vectorization by default (Matt McCline, reviewed by Deepak Jaiswal)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/78e6bfac Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/78e6bfac Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/78e6bfac Branch: refs/heads/branch-3 Commit: 78e6bfac030ab2137483dc2b97a3f58fc03c41bd Parents: 92bc9cf Author: Matt McCline <mmccl...@hortonworks.com> Authored: Wed May 9 11:06:56 2018 -0500 Committer: Matt McCline <mmccl...@hortonworks.com> Committed: Wed May 9 11:07:39 2018 -0500 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../hive/ql/exec/vector/VectorMapOperator.java | 11 +- ql/src/test/queries/clientpositive/sysdb.q | 5 + .../clientpositive/llap/jdbc_handler.q.out | 2 +- .../results/clientpositive/llap/sysdb.q.out | 526 ++++++++++++------- 5 files changed, 359 insertions(+), 187 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/78e6bfac/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 7b0fabe..3bb1e80 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3530,7 +3530,7 @@ public class HiveConf extends Configuration { "The default value is false."), HIVE_VECTORIZATION_ROW_DESERIALIZE_INPUTFORMAT_EXCLUDES( "hive.vectorized.row.serde.inputformat.excludes", - "org.apache.parquet.hadoop.ParquetInputFormat,org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat", + "org.apache.parquet.hadoop.ParquetInputFormat,org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat,org.apache.hive.storage.jdbc.JdbcInputFormat", "The input formats not supported by row deserialize vectorization."), HIVE_VECTOR_ADAPTOR_USAGE_MODE("hive.vectorized.adaptor.usage.mode", "all", new StringSet("none", "chosen", "all"), "Specifies the extent to which the VectorUDFAdaptor will be used for UDFs that do not have a corresponding vectorized class.\n" + http://git-wip-us.apache.org/repos/asf/hive/blob/78e6bfac/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java index 6f1346d..2542e03 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorMapOperator.java @@ -945,8 +945,15 @@ public class VectorMapOperator extends AbstractMapOperator { // Convert input row to standard objects. List<Object> standardObjects = new ArrayList<Object>(); - ObjectInspectorUtils.copyToStandardObject(standardObjects, deserialized, - currentPartRawRowObjectInspector, ObjectInspectorCopyOption.WRITABLE); + try { + ObjectInspectorUtils.copyToStandardObject( + standardObjects, + deserialized, + currentPartRawRowObjectInspector, + ObjectInspectorCopyOption.WRITABLE); + } catch (Exception e) { + throw new HiveException("copyToStandardObject failed: " + e); + } if (standardObjects.size() < currentDataColumnCount) { throw new HiveException("Input File Format returned row with too few columns"); } http://git-wip-us.apache.org/repos/asf/hive/blob/78e6bfac/ql/src/test/queries/clientpositive/sysdb.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/sysdb.q b/ql/src/test/queries/clientpositive/sysdb.q index 399c3ce..7f88fe1 100644 --- a/ql/src/test/queries/clientpositive/sysdb.q +++ b/ql/src/test/queries/clientpositive/sysdb.q @@ -7,6 +7,8 @@ set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager; set hive.cbo.enable=false; +-- SORT_QUERY_RESULTS + create table src_buck (key int, value string) clustered by(value) into 2 buckets; create table src_skew (key int) skewed by (key) on (1,2,3); @@ -59,6 +61,9 @@ select role_name from roles order by role_name limit 5; select principal_name, grantor from role_map order by principal_name, grantor limit 5; +explain vectorization detail +select count(*) from sds; + select count(*) from sds; select param_key, param_value from sd_params order by param_key, param_value limit 5; http://git-wip-us.apache.org/repos/asf/hive/blob/78e6bfac/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out index 94fadf1..90d88df 100644 --- a/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out +++ b/ql/src/test/results/clientpositive/llap/jdbc_handler.q.out @@ -253,7 +253,7 @@ STAGE PLANS: sort order: Statistics: Num rows: 1 Data size: 184 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: string) - Execution mode: vectorized, llap + Execution mode: llap LLAP IO: no inputs Map 4 Map Operator Tree: http://git-wip-us.apache.org/repos/asf/hive/blob/78e6bfac/ql/src/test/results/clientpositive/llap/sysdb.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out index de20f2d..3164637 100644 --- a/ql/src/test/results/clientpositive/llap/sysdb.q.out +++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out @@ -273,7 +273,7 @@ TBLPROPERTIES ( \"OWNER_NAME\", \"OWNER_TYPE\" FROM - DBS" + \"DBS\"" ) PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@DBS @@ -297,7 +297,7 @@ TBLPROPERTIES ( \"OWNER_NAME\", \"OWNER_TYPE\" FROM - DBS" + \"DBS\"" ) POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@DBS @@ -1370,7 +1370,7 @@ TBLPROPERTIES ( \"VIEW_EXPANDED_TEXT\", \"VIEW_ORIGINAL_TEXT\", \"IS_REWRITE_ENABLED\" -FROM TBLS" +FROM \"TBLS\"" ) PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@TBLS @@ -1407,7 +1407,7 @@ TBLPROPERTIES ( \"VIEW_EXPANDED_TEXT\", \"VIEW_ORIGINAL_TEXT\", \"IS_REWRITE_ENABLED\" -FROM TBLS" +FROM \"TBLS\"" ) POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@TBLS @@ -1428,7 +1428,7 @@ TBLPROPERTIES ( \"DB_NAME\", \"TBL_NAME\", \"TXN_LIST\" -FROM MV_CREATION_METADATA" +FROM \"MV_CREATION_METADATA\"" ) PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@MV_CREATION_METADATA @@ -1449,7 +1449,7 @@ TBLPROPERTIES ( \"DB_NAME\", \"TBL_NAME\", \"TXN_LIST\" -FROM MV_CREATION_METADATA" +FROM \"MV_CREATION_METADATA\"" ) POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@MV_CREATION_METADATA @@ -1465,7 +1465,7 @@ TBLPROPERTIES ( "SELECT \"MV_CREATION_METADATA_ID\", \"TBL_ID\" -FROM MV_TABLES_USED" +FROM \"MV_TABLES_USED\"" ) PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@MV_TABLES_USED @@ -1481,7 +1481,7 @@ TBLPROPERTIES ( "SELECT \"MV_CREATION_METADATA_ID\", \"TBL_ID\" -FROM MV_TABLES_USED" +FROM \"MV_TABLES_USED\"" ) POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@MV_TABLES_USED @@ -2164,16 +2164,16 @@ TBLPROPERTIES ( "hive.sql.database.type" = "METASTORE", "hive.sql.query" = "SELECT - r.NAME RP_NAME, - t.NAME NAME, - TRIGGER_EXPRESSION, - ACTION_EXPRESSION + r.\"NAME\" AS RP_NAME, + t.\"NAME\" AS NAME, + \"TRIGGER_EXPRESSION\", + \"ACTION_EXPRESSION\" FROM - WM_TRIGGER t + \"WM_TRIGGER\" t JOIN - WM_RESOURCEPLAN r + \"WM_RESOURCEPLAN\" r ON - t.RP_ID = r.RP_ID" + t.\"RP_ID\" = r.\"RP_ID\"" ) PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@WM_TRIGGERS @@ -2189,16 +2189,16 @@ TBLPROPERTIES ( "hive.sql.database.type" = "METASTORE", "hive.sql.query" = "SELECT - r.NAME RP_NAME, - t.NAME NAME, - TRIGGER_EXPRESSION, - ACTION_EXPRESSION + r.\"NAME\" AS RP_NAME, + t.\"NAME\" AS NAME, + \"TRIGGER_EXPRESSION\", + \"ACTION_EXPRESSION\" FROM - WM_TRIGGER t + \"WM_TRIGGER\" t JOIN - WM_RESOURCEPLAN r + \"WM_RESOURCEPLAN\" r ON - t.RP_ID = r.RP_ID" + t.\"RP_ID\" = r.\"RP_ID\"" ) POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@WM_TRIGGERS @@ -2215,17 +2215,17 @@ TBLPROPERTIES ( "hive.sql.database.type" = "METASTORE", "hive.sql.query" = "SELECT - WM_RESOURCEPLAN.NAME, - WM_POOL.PATH, - WM_POOL.ALLOC_FRACTION, - WM_POOL.QUERY_PARALLELISM, - WM_POOL.SCHEDULING_POLICY + \"WM_RESOURCEPLAN\".\"NAME\", + \"WM_POOL\".\"PATH\", + \"WM_POOL\".\"ALLOC_FRACTION\", + \"WM_POOL\".\"QUERY_PARALLELISM\", + \"WM_POOL\".\"SCHEDULING_POLICY\" FROM - WM_POOL + \"WM_POOL\" JOIN - WM_RESOURCEPLAN + \"WM_RESOURCEPLAN\" ON - WM_POOL.RP_ID = WM_RESOURCEPLAN.RP_ID" + \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"" ) PREHOOK: type: CREATETABLE PREHOOK: Output: SYS@WM_POOLS @@ -2242,17 +2242,17 @@ TBLPROPERTIES ( "hive.sql.database.type" = "METASTORE", "hive.sql.query" = "SELECT - WM_RESOURCEPLAN.NAME, - WM_POOL.PATH, - WM_POOL.ALLOC_FRACTION, - WM_POOL.QUERY_PARALLELISM, - WM_POOL.SCHEDULING_POLICY + \"WM_RESOURCEPLAN\".\"NAME\", + \"WM_POOL\".\"PATH\", + \"WM_POOL\".\"ALLOC_FRACTION\", + \"WM_POOL\".\"QUERY_PARALLELISM\", + \"WM_POOL\".\"SCHEDULING_POLICY\" FROM - WM_POOL + \"WM_POOL\" JOIN - WM_RESOURCEPLAN + \"WM_RESOURCEPLAN\" ON - WM_POOL.RP_ID = WM_RESOURCEPLAN.RP_ID" + \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\"" ) POSTHOOK: type: CREATETABLE POSTHOOK: Output: SYS@WM_POOLS @@ -2267,21 +2267,21 @@ TBLPROPERTIES ( "hive.sql.database.type" = "METASTORE", "hive.sql.query" = "SELECT - WM_RESOURCEPLAN.NAME RP_NAME, - WM_POOL.PATH POOL_PATH, - WM_TRIGGER.NAME TRIGGER_NAME -FROM WM_POOL_TO_TRIGGER - JOIN WM_POOL ON WM_POOL_TO_TRIGGER.POOL_ID = WM_POOL.POOL_ID - JOIN WM_TRIGGER ON WM_POOL_TO_TRIGGER.TRIGGER_ID = WM_TRIGGER.TRIGGER_ID - JOIN WM_RESOURCEPLAN ON WM_POOL.RP_ID = WM_RESOURCEPLAN.RP_ID + \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + \"WM_POOL\".\"PATH\" AS POOL_PATH, + \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME +FROM \"WM_POOL_TO_TRIGGER\" + JOIN \"WM_POOL\" ON \"WM_POOL_TO_TRIGGER\".\"POOL_ID\" = \"WM_POOL\".\"POOL_ID\" + JOIN \"WM_TRIGGER\" ON \"WM_POOL_TO_TRIGGER\".\"TRIGGER_ID\" = \"WM_TRIGGER\".\"TRIGGER_ID\" + JOIN \"WM_RESOURCEPLAN\" ON \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" UNION SELECT - WM_RESOURCEPLAN.NAME RP_NAME, - '<unmanaged queries>' POOL_PATH, - WM_TRIGGER.NAME TRIGGER_NAME -FROM WM_TRIGGER - JOIN WM_RESOURCEPLAN ON WM_TRIGGER.RP_ID = WM_RESOURCEPLAN.RP_ID -WHERE WM_TRIGGER.IS_IN_UNMANAGED = 1 + \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + '<unmanaged queries>' AS POOL_PATH, + \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME +FROM \"WM_TRIGGER\" + JOIN \"WM_RESOURCEPLAN\" ON \"WM_TRIGGER\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') " ) PREHOOK: type: CREATETABLE @@ -2297,21 +2297,21 @@ TBLPROPERTIES ( "hive.sql.database.type" = "METASTORE", "hive.sql.query" = "SELECT - WM_RESOURCEPLAN.NAME RP_NAME, - WM_POOL.PATH POOL_PATH, - WM_TRIGGER.NAME TRIGGER_NAME -FROM WM_POOL_TO_TRIGGER - JOIN WM_POOL ON WM_POOL_TO_TRIGGER.POOL_ID = WM_POOL.POOL_ID - JOIN WM_TRIGGER ON WM_POOL_TO_TRIGGER.TRIGGER_ID = WM_TRIGGER.TRIGGER_ID - JOIN WM_RESOURCEPLAN ON WM_POOL.RP_ID = WM_RESOURCEPLAN.RP_ID + \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + \"WM_POOL\".\"PATH\" AS POOL_PATH, + \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME +FROM \"WM_POOL_TO_TRIGGER\" + JOIN \"WM_POOL\" ON \"WM_POOL_TO_TRIGGER\".\"POOL_ID\" = \"WM_POOL\".\"POOL_ID\" + JOIN \"WM_TRIGGER\" ON \"WM_POOL_TO_TRIGGER\".\"TRIGGER_ID\" = \"WM_TRIGGER\".\"TRIGGER_ID\" + JOIN \"WM_RESOURCEPLAN\" ON \"WM_POOL\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" UNION SELECT - WM_RESOURCEPLAN.NAME RP_NAME, - '<unmanaged queries>' POOL_PATH, - WM_TRIGGER.NAME TRIGGER_NAME -FROM WM_TRIGGER - JOIN WM_RESOURCEPLAN ON WM_TRIGGER.RP_ID = WM_RESOURCEPLAN.RP_ID -WHERE WM_TRIGGER.IS_IN_UNMANAGED = 1 + \"WM_RESOURCEPLAN\".\"NAME\" AS RP_NAME, + '<unmanaged queries>' AS POOL_PATH, + \"WM_TRIGGER\".\"NAME\" AS TRIGGER_NAME +FROM \"WM_TRIGGER\" + JOIN \"WM_RESOURCEPLAN\" ON \"WM_TRIGGER\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +WHERE CAST(\"WM_TRIGGER\".\"IS_IN_UNMANAGED\" AS CHAR) IN ('1', 't') " ) POSTHOOK: type: CREATETABLE @@ -2329,14 +2329,14 @@ TBLPROPERTIES ( "hive.sql.database.type" = "METASTORE", "hive.sql.query" = "SELECT - WM_RESOURCEPLAN.NAME, - ENTITY_TYPE, - ENTITY_NAME, - case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end, - ORDERING -FROM WM_MAPPING -JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID -LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID + \"WM_RESOURCEPLAN\".\"NAME\", + \"ENTITY_TYPE\", + \"ENTITY_NAME\", + case when \"WM_POOL\".\"PATH\" is null then '<unmanaged>' else \"WM_POOL\".\"PATH\" end, + \"ORDERING\" +FROM \"WM_MAPPING\" +JOIN \"WM_RESOURCEPLAN\" ON \"WM_MAPPING\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_ID\" " ) PREHOOK: type: CREATETABLE @@ -2354,14 +2354,14 @@ TBLPROPERTIES ( "hive.sql.database.type" = "METASTORE", "hive.sql.query" = "SELECT - WM_RESOURCEPLAN.NAME, - ENTITY_TYPE, - ENTITY_NAME, - case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end, - ORDERING -FROM WM_MAPPING -JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID -LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID + \"WM_RESOURCEPLAN\".\"NAME\", + \"ENTITY_TYPE\", + \"ENTITY_NAME\", + case when \"WM_POOL\".\"PATH\" is null then '<unmanaged>' else \"WM_POOL\".\"PATH\" end, + \"ORDERING\" +FROM \"WM_MAPPING\" +JOIN \"WM_RESOURCEPLAN\" ON \"WM_MAPPING\".\"RP_ID\" = \"WM_RESOURCEPLAN\".\"RP_ID\" +LEFT OUTER JOIN \"WM_POOL\" ON \"WM_POOL\".\"POOL_ID\" = \"WM_MAPPING\".\"POOL_ID\" " ) POSTHOOK: type: CREATETABLE @@ -2393,18 +2393,26 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA` `DEFAULT_CHARACTER_SET_NAME`, `SQL_PATH` ) AS -SELECT +SELECT DISTINCT 'default', - `NAME`, - `OWNER_NAME`, + D.`NAME`, + D.`OWNER_NAME`, cast(null as string), cast(null as string), cast(null as string), `DB_LOCATION_URI` FROM - sys.DBS + `sys`.`DBS` D, `sys`.`TBLS` T, `sys`.`TBL_PRIVS` P +WHERE + NOT restrict_information_schema() OR + D.`DB_ID` = T.`DB_ID` + AND T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@dbs +PREHOOK: Input: sys@tbl_privs +PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@SCHEMATA PREHOOK: Output: database:information_schema POSTHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA` @@ -2417,27 +2425,35 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `SCHEMATA` `DEFAULT_CHARACTER_SET_NAME`, `SQL_PATH` ) AS -SELECT +SELECT DISTINCT 'default', - `NAME`, - `OWNER_NAME`, + D.`NAME`, + D.`OWNER_NAME`, cast(null as string), cast(null as string), cast(null as string), `DB_LOCATION_URI` FROM - sys.DBS + `sys`.`DBS` D, `sys`.`TBLS` T, `sys`.`TBL_PRIVS` P +WHERE + NOT restrict_information_schema() OR + D.`DB_ID` = T.`DB_ID` + AND T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@dbs +POSTHOOK: Input: sys@tbl_privs +POSTHOOK: Input: sys@tbls POSTHOOK: Output: INFORMATION_SCHEMA@SCHEMATA POSTHOOK: Output: database:information_schema POSTHOOK: Lineage: SCHEMATA.catalog_name SIMPLE [] POSTHOOK: Lineage: SCHEMATA.default_character_set_catalog EXPRESSION [] POSTHOOK: Lineage: SCHEMATA.default_character_set_name EXPRESSION [] POSTHOOK: Lineage: SCHEMATA.default_character_set_schema EXPRESSION [] -POSTHOOK: Lineage: SCHEMATA.schema_name SIMPLE [(dbs)dbs.FieldSchema(name:name, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: SCHEMATA.schema_name SIMPLE [(dbs)d.FieldSchema(name:name, type:string, comment:from deserializer), ] #### A masked pattern was here #### -POSTHOOK: Lineage: SCHEMATA.sql_path SIMPLE [(dbs)dbs.FieldSchema(name:db_location_uri, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: SCHEMATA.sql_path SIMPLE [(dbs)d.FieldSchema(name:db_location_uri, type:string, comment:from deserializer), ] PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES` ( `TABLE_CATALOG`, @@ -2453,7 +2469,7 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES` `IS_TYPED`, `COMMIT_ACTION` ) AS -SELECT +SELECT DISTINCT 'default', D.NAME, T.TBL_NAME, @@ -2467,11 +2483,16 @@ SELECT 'NO', cast(null as string) FROM - `sys`.`TBLS` T, `sys`.`DBS` D + `sys`.`TBLS` T, `sys`.`DBS` D, `sys`.`TBL_PRIVS` P WHERE D.`DB_ID` = T.`DB_ID` + AND (NOT restrict_information_schema() OR T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_PRIV`='SELECT') PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@dbs +PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@TABLES PREHOOK: Output: database:information_schema @@ -2490,7 +2511,7 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLES` `IS_TYPED`, `COMMIT_ACTION` ) AS -SELECT +SELECT DISTINCT 'default', D.NAME, T.TBL_NAME, @@ -2504,11 +2525,16 @@ SELECT 'NO', cast(null as string) FROM - `sys`.`TBLS` T, `sys`.`DBS` D + `sys`.`TBLS` T, `sys`.`DBS` D, `sys`.`TBL_PRIVS` P WHERE D.`DB_ID` = T.`DB_ID` + AND (NOT restrict_information_schema() OR T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_PRIV`='SELECT') POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@dbs +POSTHOOK: Input: sys@tbl_privs POSTHOOK: Input: sys@tbls POSTHOOK: Output: INFORMATION_SCHEMA@TABLES POSTHOOK: Output: database:information_schema @@ -2535,9 +2561,9 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES` `IS_GRANTABLE`, `WITH_HIERARCHY` ) AS -SELECT - `GRANTOR`, - `PRINCIPAL_NAME`, +SELECT DISTINCT + P.`GRANTOR`, + P.`PRINCIPAL_NAME`, 'default', D.`NAME`, T.`TBL_NAME`, @@ -2545,12 +2571,18 @@ SELECT IF (P.`GRANT_OPTION` == 0, 'NO', 'YES'), 'NO' FROM - sys.`TBL_PRIVS` P, - sys.`TBLS` T, - sys.`DBS` D + `sys`.`TBL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_PRIVS` P2 WHERE - P.TBL_ID = T.TBL_ID - AND T.DB_ID = D.DB_ID + P.`TBL_ID` = T.`TBL_ID` + AND T.`DB_ID` = D.`DB_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) + AND P2.`TBL_PRIV`='SELECT') PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@dbs PREHOOK: Input: sys@tbl_privs @@ -2568,9 +2600,9 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `TABLE_PRIVILEGES` `IS_GRANTABLE`, `WITH_HIERARCHY` ) AS -SELECT - `GRANTOR`, - `PRINCIPAL_NAME`, +SELECT DISTINCT + P.`GRANTOR`, + P.`PRINCIPAL_NAME`, 'default', D.`NAME`, T.`TBL_NAME`, @@ -2578,12 +2610,18 @@ SELECT IF (P.`GRANT_OPTION` == 0, 'NO', 'YES'), 'NO' FROM - sys.`TBL_PRIVS` P, - sys.`TBLS` T, - sys.`DBS` D + `sys`.`TBL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_PRIVS` P2 WHERE - P.TBL_ID = T.TBL_ID - AND T.DB_ID = D.DB_ID + P.`TBL_ID` = T.`TBL_ID` + AND T.`DB_ID` = D.`DB_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) + AND P2.`TBL_PRIV`='SELECT') POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@dbs POSTHOOK: Input: sys@tbl_privs @@ -2648,7 +2686,7 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS` `DECLARED_NUMERIC_PRECISION`, `DECLARED_NUMERIC_SCALE` ) AS -SELECT +SELECT DISTINCT 'default', D.NAME, T.TBL_NAME, @@ -2737,18 +2775,26 @@ SELECT WHEN lower(C.TYPE_NAME) like 'numeric%' THEN 10 ELSE null END FROM - sys.`COLUMNS_V2` C, - sys.`SDS` S, - sys.`TBLS` T, - sys.`DBS` D + `sys`.`COLUMNS_V2` C, + `sys`.`SDS` S, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_COL_PRIVS` P WHERE S.`SD_ID` = T.`SD_ID` AND T.`DB_ID` = D.`DB_ID` AND C.`CD_ID` = S.`CD_ID` + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` + AND C.`COLUMN_NAME` = P.`COLUMN_NAME` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_COL_PRIV`='SELECT') PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@columns_v2 PREHOOK: Input: sys@dbs PREHOOK: Input: sys@sds +PREHOOK: Input: sys@tbl_col_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@COLUMNS PREHOOK: Output: database:information_schema @@ -2802,7 +2848,7 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMNS` `DECLARED_NUMERIC_PRECISION`, `DECLARED_NUMERIC_SCALE` ) AS -SELECT +SELECT DISTINCT 'default', D.NAME, T.TBL_NAME, @@ -2891,18 +2937,26 @@ SELECT WHEN lower(C.TYPE_NAME) like 'numeric%' THEN 10 ELSE null END FROM - sys.`COLUMNS_V2` C, - sys.`SDS` S, - sys.`TBLS` T, - sys.`DBS` D + `sys`.`COLUMNS_V2` C, + `sys`.`SDS` S, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`TBL_COL_PRIVS` P WHERE S.`SD_ID` = T.`SD_ID` AND T.`DB_ID` = D.`DB_ID` AND C.`CD_ID` = S.`CD_ID` + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` + AND C.`COLUMN_NAME` = P.`COLUMN_NAME` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_COL_PRIV`='SELECT') POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@columns_v2 POSTHOOK: Input: sys@dbs POSTHOOK: Input: sys@sds +POSTHOOK: Input: sys@tbl_col_privs POSTHOOK: Input: sys@tbls POSTHOOK: Output: INFORMATION_SCHEMA@COLUMNS POSTHOOK: Output: database:information_schema @@ -2964,32 +3018,35 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES` `PRIVILEGE_TYPE`, `IS_GRANTABLE` ) AS -SELECT - `GRANTOR`, - `PRINCIPAL_NAME`, +SELECT DISTINCT + P.`GRANTOR`, + P.`PRINCIPAL_NAME`, 'default', D.`NAME`, T.`TBL_NAME`, - C.`COLUMN_NAME`, + P.`COLUMN_NAME`, P.`TBL_COL_PRIV`, IF (P.`GRANT_OPTION` == 0, 'NO', 'YES') FROM - sys.`TBL_COL_PRIVS` P, - sys.`TBLS` T, - sys.`DBS` D, - sys.`COLUMNS_V2` C, - sys.`SDS` S + `sys`.`TBL_COL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`SDS` S, + `sys`.`TBL_PRIVS` P2 WHERE S.`SD_ID` = T.`SD_ID` AND T.`DB_ID` = D.`DB_ID` AND P.`TBL_ID` = T.`TBL_ID` - AND P.`COLUMN_NAME` = C.`COLUMN_NAME` - AND C.`CD_ID` = S.`CD_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) + AND P2.`TBL_PRIV`='SELECT') PREHOOK: type: CREATEVIEW -PREHOOK: Input: sys@columns_v2 PREHOOK: Input: sys@dbs PREHOOK: Input: sys@sds PREHOOK: Input: sys@tbl_col_privs +PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@COLUMN_PRIVILEGES PREHOOK: Output: database:information_schema @@ -3004,36 +3061,39 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `COLUMN_PRIVILEGES` `PRIVILEGE_TYPE`, `IS_GRANTABLE` ) AS -SELECT - `GRANTOR`, - `PRINCIPAL_NAME`, +SELECT DISTINCT + P.`GRANTOR`, + P.`PRINCIPAL_NAME`, 'default', D.`NAME`, T.`TBL_NAME`, - C.`COLUMN_NAME`, + P.`COLUMN_NAME`, P.`TBL_COL_PRIV`, IF (P.`GRANT_OPTION` == 0, 'NO', 'YES') FROM - sys.`TBL_COL_PRIVS` P, - sys.`TBLS` T, - sys.`DBS` D, - sys.`COLUMNS_V2` C, - sys.`SDS` S + `sys`.`TBL_COL_PRIVS` P, + `sys`.`TBLS` T, + `sys`.`DBS` D, + `sys`.`SDS` S, + `sys`.`TBL_PRIVS` P2 WHERE S.`SD_ID` = T.`SD_ID` AND T.`DB_ID` = D.`DB_ID` AND P.`TBL_ID` = T.`TBL_ID` - AND P.`COLUMN_NAME` = C.`COLUMN_NAME` - AND C.`CD_ID` = S.`CD_ID` + AND (NOT restrict_information_schema() OR + P.`TBL_ID` = P2.`TBL_ID` AND P.`PRINCIPAL_NAME` = P2.`PRINCIPAL_NAME` AND P.`PRINCIPAL_TYPE` = P2.`PRINCIPAL_TYPE` + AND (P2.`PRINCIPAL_NAME`=current_user() AND P2.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P2.`PRINCIPAL_NAME`) OR P2.`PRINCIPAL_NAME` = 'public') AND P2.`PRINCIPAL_TYPE`='GROUP')) + AND P2.`TBL_PRIV`='SELECT') POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: sys@columns_v2 POSTHOOK: Input: sys@dbs POSTHOOK: Input: sys@sds POSTHOOK: Input: sys@tbl_col_privs +POSTHOOK: Input: sys@tbl_privs POSTHOOK: Input: sys@tbls POSTHOOK: Output: INFORMATION_SCHEMA@COLUMN_PRIVILEGES POSTHOOK: Output: database:information_schema -POSTHOOK: Lineage: COLUMN_PRIVILEGES.column_name SIMPLE [(columns_v2)c.FieldSchema(name:column_name, type:string, comment:from deserializer), ] +POSTHOOK: Lineage: COLUMN_PRIVILEGES.column_name SIMPLE [(tbl_col_privs)p.FieldSchema(name:column_name, type:string, comment:from deserializer), ] POSTHOOK: Lineage: COLUMN_PRIVILEGES.grantee SIMPLE [(tbl_col_privs)p.FieldSchema(name:principal_name, type:string, comment:from deserializer), ] POSTHOOK: Lineage: COLUMN_PRIVILEGES.grantor SIMPLE [(tbl_col_privs)p.FieldSchema(name:grantor, type:string, comment:from deserializer), ] POSTHOOK: Lineage: COLUMN_PRIVILEGES.is_grantable EXPRESSION [(tbl_col_privs)p.FieldSchema(name:grant_option, type:int, comment:from deserializer), ] @@ -3054,7 +3114,7 @@ PREHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS` `IS_TRIGGER_DELETABLE`, `IS_TRIGGER_INSERTABLE_INTO` ) AS -SELECT +SELECT DISTINCT 'default', D.NAME, T.TBL_NAME, @@ -3067,12 +3127,19 @@ SELECT false FROM `sys`.`DBS` D, - `sys`.`TBLS` T + `sys`.`TBLS` T, + `sys`.`TBL_PRIVS` P WHERE - D.`DB_ID` = T.`DB_ID` AND - length(T.VIEW_ORIGINAL_TEXT) > 0 + D.`DB_ID` = T.`DB_ID` + AND length(T.VIEW_ORIGINAL_TEXT) > 0 + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_PRIV`='SELECT') PREHOOK: type: CREATEVIEW PREHOOK: Input: sys@dbs +PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls PREHOOK: Output: INFORMATION_SCHEMA@VIEWS PREHOOK: Output: database:information_schema @@ -3089,7 +3156,7 @@ POSTHOOK: query: CREATE VIEW IF NOT EXISTS `VIEWS` `IS_TRIGGER_DELETABLE`, `IS_TRIGGER_INSERTABLE_INTO` ) AS -SELECT +SELECT DISTINCT 'default', D.NAME, T.TBL_NAME, @@ -3102,12 +3169,19 @@ SELECT false FROM `sys`.`DBS` D, - `sys`.`TBLS` T + `sys`.`TBLS` T, + `sys`.`TBL_PRIVS` P WHERE - D.`DB_ID` = T.`DB_ID` AND - length(T.VIEW_ORIGINAL_TEXT) > 0 + D.`DB_ID` = T.`DB_ID` + AND length(T.VIEW_ORIGINAL_TEXT) > 0 + AND (NOT restrict_information_schema() OR + T.`TBL_ID` = P.`TBL_ID` + AND (P.`PRINCIPAL_NAME`=current_user() AND P.`PRINCIPAL_TYPE`='USER' + OR ((array_contains(current_groups(), P.`PRINCIPAL_NAME`) OR P.`PRINCIPAL_NAME` = 'public') AND P.`PRINCIPAL_TYPE`='GROUP')) + AND P.`TBL_PRIV`='SELECT') POSTHOOK: type: CREATEVIEW POSTHOOK: Input: sys@dbs +POSTHOOK: Input: sys@tbl_privs POSTHOOK: Input: sys@tbls POSTHOOK: Output: INFORMATION_SCHEMA@VIEWS POSTHOOK: Output: database:information_schema @@ -3256,6 +3330,95 @@ POSTHOOK: query: select principal_name, grantor from role_map order by principal POSTHOOK: type: QUERY POSTHOOK: Input: sys@role_map #### A masked pattern was here #### +PREHOOK: query: explain vectorization detail +select count(*) from sds +PREHOOK: type: QUERY +POSTHOOK: query: explain vectorization detail +select count(*) from sds +POSTHOOK: type: QUERY +PLAN VECTORIZATION: + enabled: true + enabledConditionsMet: [hive.vectorized.execution.enabled IS true] + +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: sds + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Select Operator + Statistics: Num rows: 1 Data size: 0 Basic stats: PARTIAL Column stats: COMPLETE + Group By Operator + aggregations: count() + mode: hash + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + Reduce Output Operator + sort order: + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + value expressions: _col0 (type: bigint) + Execution mode: llap + LLAP IO: no inputs + Map Vectorization: + enabled: false + enabledConditionsNotMet: hive.vectorized.use.row.serde.deserialize IS true AND hive.vectorized.row.serde.inputformat.excludes NOT CONTAINS org.apache.hive.storage.jdbc.JdbcInputFormat IS false, hive.vectorized.use.row.serde.deserialize IS false + inputFileFormats: org.apache.hive.storage.jdbc.JdbcInputFormat + Reducer 2 + Execution mode: vectorized, llap + Reduce Vectorization: + enabled: true + enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez, spark] IS true + reduceColumnNullOrder: + reduceColumnSortOrder: + allNative: false + usesVectorUDFAdaptor: false + vectorized: true + rowBatchContext: + dataColumnCount: 1 + dataColumns: VALUE._col0:bigint + partitionColumnCount: 0 + scratchColumnTypeNames: [] + Reduce Operator Tree: + Group By Operator + aggregations: count(VALUE._col0) + Group By Vectorization: + aggregators: VectorUDAFCountMerge(col 0:bigint) -> bigint + className: VectorGroupByOperator + groupByMode: MERGEPARTIAL + native: false + vectorProcessingMode: GLOBAL + projectedOutputColumnNums: [0] + mode: mergepartial + outputColumnNames: _col0 + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + File Output Operator + compressed: false + File Sink Vectorization: + className: VectorFileSinkOperator + native: false + Statistics: Num rows: 1 Data size: 8 Basic stats: PARTIAL Column stats: COMPLETE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + PREHOOK: query: select count(*) from sds PREHOOK: type: QUERY PREHOOK: Input: sys@sds @@ -3283,9 +3446,9 @@ POSTHOOK: Input: sys@sequence_table #### A masked pattern was here #### org.apache.hadoop.hive.metastore.model.MCatalog org.apache.hadoop.hive.metastore.model.MColumnDescriptor -org.apache.hadoop.hive.metastore.model.MCreationMetadata org.apache.hadoop.hive.metastore.model.MDatabase org.apache.hadoop.hive.metastore.model.MFunction +org.apache.hadoop.hive.metastore.model.MGlobalPrivilege PREHOOK: query: select name, slib from serdes order by name, slib limit 5 PREHOOK: type: QUERY PREHOOK: Input: sys@serdes @@ -3421,8 +3584,8 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@tab_col_stats #### A masked pattern was here #### alltypesorc cbigint 3115 6064 -alltypesorc cboolean1 3114 NULL -alltypesorc cboolean2 3115 NULL +alltypesorc cboolean1 3114 0 +alltypesorc cboolean2 3115 0 alltypesorc cdouble 3114 5569 alltypesorc cfloat 3115 131 alltypesorc cint 3115 6029 @@ -3479,11 +3642,11 @@ POSTHOOK: type: QUERY POSTHOOK: Input: sys@table_params POSTHOOK: Input: sys@table_stats_view #### A masked pattern was here #### -{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} 0 0 0 0 -{"BASIC_STATS":"true","COLUMN_STATS":{"pool_path":"true","rp_name":"true","trigger_name":"true"}} 0 0 0 0 -{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true","value":"true"}} 0 0 0 0 -{"BASIC_STATS":"true","COLUMN_STATS":{"a":"true","b":"true","c":"true","d":"true","e":"true","f":"true","g":"true"}} 0 0 0 0 -{"BASIC_STATS":"true","COLUMN_STATS":{"create_time":"true","last_access_time":"true","part_id":"true","part_name":"true","sd_id":"true","tbl_id":"true"}} 0 0 0 0 +{"BASIC_STATS":"true","COLUMN_STATS":{"action_expression":"true","name":"true","rp_name":"true","trigger_expression":"true"}} 0 0 0 0 +#### A masked pattern was here #### +{"BASIC_STATS":"true","COLUMN_STATS":{"db_id":"true","param_key":"true","param_value":"true"}} 0 0 0 0 +{"BASIC_STATS":"true","COLUMN_STATS":{"key":"true"}} 0 0 0 0 +{"BASIC_STATS":"true","COLUMN_STATS":{"param_key":"true","param_value":"true","serde_id":"true"}} 0 0 0 0 PREHOOK: query: select COLUMN_STATS_ACCURATE, NUM_FILES, NUM_ROWS, RAW_DATA_SIZE, TOTAL_SIZE FROM PARTITION_STATS_VIEW where COLUMN_STATS_ACCURATE is not null order by NUM_FILES, NUM_ROWS, RAW_DATA_SIZE limit 5 PREHOOK: type: QUERY PREHOOK: Input: sys@partition_params @@ -3556,7 +3719,7 @@ STAGE PLANS: Execution mode: llap LLAP IO: no inputs Reducer 2 - Execution mode: llap + Execution mode: vectorized, llap Reduce Operator Tree: Group By Operator aggregations: max(VALUE._col0) @@ -3592,27 +3755,35 @@ PREHOOK: Input: database:information_schema POSTHOOK: query: use INFORMATION_SCHEMA POSTHOOK: type: SWITCHDATABASE POSTHOOK: Input: database:information_schema +Warning: Shuffle Join MERGEJOIN[20][tables = [d, t, p]] in Stage 'Reducer 2' is a cross product PREHOOK: query: select count(*) from SCHEMATA PREHOOK: type: QUERY PREHOOK: Input: information_schema@schemata PREHOOK: Input: sys@dbs +PREHOOK: Input: sys@tbl_privs +PREHOOK: Input: sys@tbls #### A masked pattern was here #### POSTHOOK: query: select count(*) from SCHEMATA POSTHOOK: type: QUERY POSTHOOK: Input: information_schema@schemata POSTHOOK: Input: sys@dbs +POSTHOOK: Input: sys@tbl_privs +POSTHOOK: Input: sys@tbls #### A masked pattern was here #### 3 +Warning: Shuffle Join MERGEJOIN[38][tables = [t, d, p]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select * from TABLES order by TABLE_SCHEMA, TABLE_NAME PREHOOK: type: QUERY PREHOOK: Input: information_schema@tables PREHOOK: Input: sys@dbs +PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls #### A masked pattern was here #### POSTHOOK: query: select * from TABLES order by TABLE_SCHEMA, TABLE_NAME POSTHOOK: type: QUERY POSTHOOK: Input: information_schema@tables POSTHOOK: Input: sys@dbs +POSTHOOK: Input: sys@tbl_privs POSTHOOK: Input: sys@tbls #### A masked pattern was here #### default default alltypesorc BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL @@ -3672,6 +3843,7 @@ default sys wm_pools BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys wm_pools_to_triggers BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys wm_resourceplans BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL default sys wm_triggers BASE_TABLE NULL NULL NULL NULL NULL YES NO NULL +Warning: Shuffle Join MERGEJOIN[60][tables = [p, t, d, p2]] in Stage 'Reducer 4' is a cross product PREHOOK: query: select * from TABLE_PRIVILEGES order by GRANTOR, GRANTEE, TABLE_SCHEMA, TABLE_NAME, PRIVILEGE_TYPE limit 10 PREHOOK: type: QUERY PREHOOK: Input: information_schema@table_privileges @@ -3696,12 +3868,14 @@ hive_test_user hive_test_user default default moretypes SELECT YES NO hive_test_user hive_test_user default default moretypes UPDATE YES NO hive_test_user hive_test_user default default scr_txn DELETE YES NO hive_test_user hive_test_user default default scr_txn INSERT YES NO +Warning: Shuffle Join MERGEJOIN[82][tables = [c, s, t, d, p]] in Stage 'Reducer 5' is a cross product PREHOOK: query: select * from COLUMNS where TABLE_NAME = 'alltypesorc' or TABLE_NAME = 'moretypes' order by TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION PREHOOK: type: QUERY PREHOOK: Input: information_schema@columns PREHOOK: Input: sys@columns_v2 PREHOOK: Input: sys@dbs PREHOOK: Input: sys@sds +PREHOOK: Input: sys@tbl_col_privs PREHOOK: Input: sys@tbls #### A masked pattern was here #### POSTHOOK: query: select * from COLUMNS where TABLE_NAME = 'alltypesorc' or TABLE_NAME = 'moretypes' order by TABLE_SCHEMA, TABLE_NAME, ORDINAL_POSITION @@ -3710,55 +3884,41 @@ POSTHOOK: Input: information_schema@columns POSTHOOK: Input: sys@columns_v2 POSTHOOK: Input: sys@dbs POSTHOOK: Input: sys@sds +POSTHOOK: Input: sys@tbl_col_privs POSTHOOK: Input: sys@tbls #### A masked pattern was here #### -default default alltypesorc ctinyint 0 NULL YES tinyint NULL NULL 3 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES tinyint 3 10 -default default alltypesorc csmallint 1 NULL YES smallint NULL NULL 5 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES smallint 5 10 -default default alltypesorc cint 2 NULL YES int NULL NULL 10 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES int 10 10 -default default alltypesorc cbigint 3 NULL YES bigint NULL NULL 19 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES bigint 19 10 -default default alltypesorc cfloat 4 NULL YES float NULL NULL 23 2 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES float 23 2 -default default alltypesorc cdouble 5 NULL YES double NULL NULL 53 2 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES double 53 2 -default default alltypesorc cstring1 6 NULL YES string NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES string NULL NULL -default default alltypesorc cstring2 7 NULL YES string NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES string NULL NULL -default default alltypesorc ctimestamp1 8 NULL YES timestamp NULL NULL NULL NULL NULL 9 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES timestamp NULL NULL -default default alltypesorc ctimestamp2 9 NULL YES timestamp NULL NULL NULL NULL NULL 9 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES timestamp NULL NULL -default default alltypesorc cboolean1 10 NULL YES boolean NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES boolean NULL NULL -default default alltypesorc cboolean2 11 NULL YES boolean NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 12 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES boolean NULL NULL -default default moretypes a 0 NULL YES decimal(10,2) NULL NULL 10 10 2 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES decimal(10,2) 10 10 -default default moretypes b 1 NULL YES tinyint NULL NULL 3 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES tinyint 3 10 -default default moretypes c 2 NULL YES smallint NULL NULL 5 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES smallint 5 10 -default default moretypes d 3 NULL YES int NULL NULL 10 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES int 10 10 -default default moretypes e 4 NULL YES bigint NULL NULL 19 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES bigint 19 10 -default default moretypes f 5 NULL YES varchar(10) 10 10 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES varchar(10) NULL NULL -default default moretypes g 6 NULL YES char(3) 3 3 NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL NULL 16 NO NO NULL NULL NULL NULL NULL NULL NEVER NULL NO NO NULL YES char(3) NULL NULL +Warning: Shuffle Join MERGEJOIN[82][tables = [p, t, d, s, p2]] in Stage 'Reducer 5' is a cross product PREHOOK: query: select * from COLUMN_PRIVILEGES order by GRANTOR, GRANTEE, TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME limit 10 PREHOOK: type: QUERY PREHOOK: Input: information_schema@column_privileges -PREHOOK: Input: sys@columns_v2 PREHOOK: Input: sys@dbs PREHOOK: Input: sys@sds PREHOOK: Input: sys@tbl_col_privs +PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls #### A masked pattern was here #### POSTHOOK: query: select * from COLUMN_PRIVILEGES order by GRANTOR, GRANTEE, TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME limit 10 POSTHOOK: type: QUERY POSTHOOK: Input: information_schema@column_privileges -POSTHOOK: Input: sys@columns_v2 POSTHOOK: Input: sys@dbs POSTHOOK: Input: sys@sds POSTHOOK: Input: sys@tbl_col_privs +POSTHOOK: Input: sys@tbl_privs POSTHOOK: Input: sys@tbls #### A masked pattern was here #### +Warning: Shuffle Join MERGEJOIN[38][tables = [d, t, p]] in Stage 'Reducer 3' is a cross product PREHOOK: query: select TABLE_SCHEMA, TABLE_NAME from views order by TABLE_SCHEMA, TABLE_NAME PREHOOK: type: QUERY PREHOOK: Input: information_schema@views PREHOOK: Input: sys@dbs +PREHOOK: Input: sys@tbl_privs PREHOOK: Input: sys@tbls #### A masked pattern was here #### POSTHOOK: query: select TABLE_SCHEMA, TABLE_NAME from views order by TABLE_SCHEMA, TABLE_NAME POSTHOOK: type: QUERY POSTHOOK: Input: information_schema@views POSTHOOK: Input: sys@dbs +POSTHOOK: Input: sys@tbl_privs POSTHOOK: Input: sys@tbls #### A masked pattern was here #### information_schema column_privileges
