[
https://issues.apache.org/jira/browse/HIVE-24596?focusedWorklogId=587026&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-587026
]
ASF GitHub Bot logged work on HIVE-24596:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 22/Apr/21 03:29
Start Date: 22/Apr/21 03:29
Worklog Time Spent: 10m
Work Description: rbalamohan commented on a change in pull request #2033:
URL: https://github.com/apache/hive/pull/2033#discussion_r618054976
##########
File path: ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
##########
@@ -0,0 +1,1019 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.exec;
+
+import com.google.common.base.Joiner;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hive.common.StatsSetupConst;
+import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData;
+import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj;
+import org.apache.hadoop.hive.metastore.api.DateColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.SerDeInfo;
+import org.apache.hadoop.hive.metastore.api.SkewedInfo;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+import org.apache.hadoop.hive.ql.ddl.ShowUtils;
+import org.apache.hadoop.hive.ql.ddl.table.create.CreateTableOperation;
+import org.apache.hadoop.hive.ql.metadata.CheckConstraint;
+import org.apache.hadoop.hive.ql.metadata.CheckConstraint.CheckConstraintCol;
+import org.apache.hadoop.hive.ql.metadata.DefaultConstraint;
+import
org.apache.hadoop.hive.ql.metadata.DefaultConstraint.DefaultConstraintCol;
+import org.apache.hadoop.hive.ql.metadata.ForeignKeyInfo;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.NotNullConstraint;
+import org.apache.hadoop.hive.ql.metadata.Partition;
+import org.apache.hadoop.hive.ql.metadata.PrimaryKeyInfo;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.apache.hadoop.hive.ql.metadata.UniqueConstraint;
+import org.apache.hadoop.hive.ql.util.DirectionUtils;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
+import org.apache.hive.common.util.HiveStringUtils;
+import org.stringtemplate.v4.ST;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
+import static
org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE;
+
+public class DDLPlanUtils {
+ private static final String EXTERNAL = "external";
+ private static final String TEMPORARY = "temporary";
+ private static final String LIST_COLUMNS = "columns";
+ private static final String COMMENT = "comment";
+ private static final String PARTITIONS = "partitions";
+ private static final String BUCKETS = "buckets";
+ private static final String SKEWED = "skewedinfo";
+ private static final String ROW_FORMAT = "row_format";
+ private static final String LOCATION_BLOCK = "location_block";
+ private static final String LOCATION = "location";
+ private static final String PROPERTIES = "properties";
+ private static final String TABLE_NAME = "TABLE_NAME";
+ private static final String DATABASE_NAME = "DATABASE_NAME";
+ private static final String DATABASE_NAME_FR = "DATABASE_NAME_FR";
+ private static final String PARTITION = "PARTITION";
+ private static final String COLUMN_NAME = "COLUMN_NAME";
+ private static final String TBLPROPERTIES = "TABLE_PROPERTIES";
+ private static final String PARTITION_NAME = "PARTITION_NAME";
+ private static final String CONSTRAINT_NAME = "CONSTRAINT_NAME";
+ private static final String COL_NAMES = "COLUMN_NAMES";
+ private static final String CHILD_TABLE_NAME = "CHILD_TABLE_NAME";
+ private static final String PARENT_TABLE_NAME = "PARENT_TABLE_NAME";
+ private static final String CHILD_COL_NAME = "CHILD_COL_NAME";
+ private static final String PARENT_COL_NAME = "PARENT_COL_NAME";
+ private static final String CHECK_EXPRESSION = "CHECK_EXPRESSION";
+ private static final String DEFAULT_VALUE = "DEFAULT_VALUE";
+ private static final String COL_TYPE = "COL_TYPE";
+ private static final String SQL = "SQL";
+ private static final String COMMENT_SQL = "COMMENT_SQL";
+ private static final String HIVE_DEFAULT_PARTITION =
"__HIVE_DEFAULT_PARTITION__";
+ private static final String BASE_64_VALUE = "BASE_64";
+ private static final String explain = "EXPLAIN";
+ private static final String numNulls = "'numNulls'='";
+ private static final String numDVs = "'numDVs'='";
+ private static final String numTrues = "'numTrues'='";
+ private static final String numFalses = "'numFalses'='";
+ private static final String lowValue = "'lowValue'='";
+ private static final String highValue = "'highValue'='";
+ private static final String avgColLen = "'avgColLen'='";
+ private static final String maxColLen = "'maxColLen'='";
+ private static final String[] req = {"numRows", "rawDataSize"};
+ private static final String[] explain_plans = {" ", " CBO ", " VECTORIZED "};
+
+ private static final String CREATE_DATABASE_STMT = "CREATE DATABASE IF NOT
EXISTS <" + DATABASE_NAME + ">;";
+
+ private final String CREATE_TABLE_TEMPLATE =
+ "CREATE <" + TEMPORARY + "><" + EXTERNAL + ">TABLE <if(" + DATABASE_NAME
+ ")>`<" + DATABASE_NAME + ">`.<endif>"
+ + "`<" + TABLE_NAME + ">`(\n" +
+ "<" + LIST_COLUMNS + ">)\n" +
+ "<" + COMMENT + ">\n" +
+ "<" + PARTITIONS + ">\n" +
+ "<" + BUCKETS + ">\n" +
+ "<" + SKEWED + ">\n" +
+ "<" + ROW_FORMAT + ">\n" +
+ "<" + LOCATION_BLOCK + ">" +
+ "TBLPROPERTIES (\n" +
+ "<" + PROPERTIES + ">)";
+
+ private static final String CREATE_VIEW_TEMPLATE =
+ "CREATE VIEW <if(" + DATABASE_NAME + ")>`<" + DATABASE_NAME +
">`.<endif>`<" + TABLE_NAME +
+ ">`<" + PARTITIONS + "> AS <" + SQL +">";
+
+ private final String CREATE_TABLE_TEMPLATE_LOCATION = "LOCATION\n" +
+ "<" + LOCATION + ">\n";
+
+ private final Set<String> PROPERTIES_TO_IGNORE_AT_TBLPROPERTIES = Sets.union(
+ ImmutableSet.of("TEMPORARY", "EXTERNAL", "comment",
"SORTBUCKETCOLSPREFIX", META_TABLE_STORAGE),
+ new HashSet<String>(StatsSetupConst.TABLE_PARAMS_STATS_KEYS));
+
+ private final String ALTER_TABLE_CREATE_PARTITION = "<if(" + COMMENT_SQL +
")><" + COMMENT_SQL + "> <endif>" + "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME +
+ "> ADD IF NOT EXISTS PARTITION (<" + PARTITION +
+ ">);";
+
+ private final String ALTER_TABLE_UPDATE_STATISTICS_TABLE_COLUMN = "ALTER
TABLE <"
+ + DATABASE_NAME + ">.<" +
+ TABLE_NAME + "> UPDATE STATISTICS FOR COLUMN <"
+ + COLUMN_NAME + "> SET(<" + TBLPROPERTIES + "> );";
+
+ private final String ALTER_TABLE_UPDATE_STATISTICS_PARTITION_COLUMN = "<if("
+ COMMENT_SQL + ")><" + COMMENT_SQL + "> <endif>" + "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME +
+ "> PARTITION (<" + PARTITION_NAME +
+ ">) UPDATE STATISTICS FOR COLUMN <"
+ + COLUMN_NAME + "> SET(<" + TBLPROPERTIES + "> );";
+
+ private final String ALTER_TABLE_UPDATE_STATISTICS_TABLE_BASIC = "ALTER
TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME +
+ "> UPDATE STATISTICS SET(<" + TBLPROPERTIES + "> );";
+
+ private final String ALTER_TABLE_UPDATE_STATISTICS_PARTITION_BASIC = "<if("
+ COMMENT_SQL + ")><" + COMMENT_SQL + "> <endif>" + "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME + "> PARTITION (<" +
+ PARTITION_NAME + ">) UPDATE STATISTICS SET(<" + TBLPROPERTIES + "> );";
+ private final String ALTER_TABLE_ADD_PRIMARY_KEY = "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME + "> ADD CONSTRAINT <" +
+ CONSTRAINT_NAME + "> PRIMARY KEY (<" + COL_NAMES + ">) DISABLE
NOVALIDATE;";
+
+ private final String ALTER_TABLE_ADD_FOREIGN_KEY = "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + CHILD_TABLE_NAME + "> ADD CONSTRAINT <"
+ + CONSTRAINT_NAME + "> FOREIGN KEY (<" + CHILD_COL_NAME + ">) REFERENCES
<"
+ + DATABASE_NAME_FR + ">.<" + PARENT_TABLE_NAME + ">(<" + PARENT_COL_NAME
+ ">) DISABLE NOVALIDATE RELY;";
+
+ private final String ALTER_TABLE_ADD_UNIQUE_CONSTRAINT = "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME + "> ADD CONSTRAINT <" +
+ CONSTRAINT_NAME + "> UNIQUE (<" + COLUMN_NAME + ">) DISABLE NOVALIDATE;";
+
+ private final String ALTER_TABLE_ADD_CHECK_CONSTRAINT = "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME +
+ "> ADD CONSTRAINT <" + CONSTRAINT_NAME + "> CHECK (<" +
+ CHECK_EXPRESSION + ">) DISABLE;";
+
+ private final String ALTER_TABLE_ADD_NOT_NULL_CONSTRAINT = "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME + "> CHANGE COLUMN < "
+ + COLUMN_NAME + "> <" + COLUMN_NAME +
+ "> <" + COL_TYPE + "> CONSTRAINT <" + CONSTRAINT_NAME + "> NOT NULL
DISABLE;";
+
+ private final String ALTER_TABLE_ADD_DEFAULT_CONSTRAINT = "ALTER TABLE <"
+ + DATABASE_NAME + ">.<" + TABLE_NAME + "> CHANGE COLUMN < "
+ + COLUMN_NAME + "> <" + COLUMN_NAME +
+ "> <" + COL_TYPE + "> CONSTRAINT <" + CONSTRAINT_NAME + "> DEFAULT <" +
DEFAULT_VALUE + "> DISABLE;";
+
+ private final String EXIST_BIT_VECTORS = "-- BIT VECTORS PRESENT FOR <" +
DATABASE_NAME + ">.<" + TABLE_NAME + "> " +
+ "FOR COLUMN <" + COLUMN_NAME + "> BUT THEY ARE NOT SUPPORTED YET. THE
BASE64 VALUE FOR THE BITVECTOR IS <" +
+ BASE_64_VALUE +"> ";
+
+ private final String EXIST_BIT_VECTORS_PARTITIONED = "-- BIT VECTORS PRESENT
FOR <" + DATABASE_NAME + ">.<" +
+ TABLE_NAME + "> PARTITION <" + PARTITION_NAME + "> FOR COLUMN <"
+ + COLUMN_NAME + "> BUT THEY ARE NOT SUPPORTED YET.THE BASE64 VALUE FOR
THE BITVECTOR IS <" +
+ BASE_64_VALUE +"> ";
+
+ /**
+ * Returns the create database query for a give database name.
+ *
+ * @param dbNames
+ * @return
+ */
+ public List<String> getCreateDatabaseStmt(Set<String> dbNames) {
+ List<String> allDbStmt = new ArrayList<String>();
+ for (String dbName : dbNames) {
+ if (dbName.equals("default")) {
+ continue;
+ }
+ ST command = new ST(CREATE_DATABASE_STMT);
+ command.add(DATABASE_NAME, dbName);
+ allDbStmt.add(command.render());
+ }
+ return allDbStmt;
+ }
+
+ public Map<String, String> getTableColumnsToType(Table tbl) {
+ List<FieldSchema> fieldSchemas = tbl.getAllCols();
+ Map<String, String> ret = new HashMap<String, String>();
+ fieldSchemas.stream().forEach(f -> ret.put(f.getName(), f.getType()));
+ return ret;
+ }
+
+ public List<String> getTableColumnNames(Table tbl) {
+ List<FieldSchema> fieldSchemas = tbl.getAllCols();
+ List<String> ret = new ArrayList<String>();
+ fieldSchemas.stream().forEach(f -> ret.add(f.getName()));
+ return ret;
+ }
+
+ public String getPartitionActualName(Partition pt) {
+ Map<String, String> colTypeMap = getTableColumnsToType(pt.getTable());
+ String[] partColsDef = pt.getName().split(",");
+ List<String> ptParam = new ArrayList<>();
+ for (String partCol : partColsDef) {
+ String[] colValue = partCol.split("=");
+ if (colTypeMap.get(colValue[0]).equals("string")) {
Review comment:
equalsIgnoreCase?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
Issue Time Tracking
-------------------
Worklog Id: (was: 587026)
Time Spent: 1h 20m (was: 1h 10m)
> Explain ddl for debugging
> -------------------------
>
> Key: HIVE-24596
> URL: https://issues.apache.org/jira/browse/HIVE-24596
> Project: Hive
> Issue Type: Improvement
> Reporter: Rajesh Balamohan
> Assignee: Harshit Gupta
> Priority: Major
> Labels: pull-request-available
> Attachments: output, query, table_definitions
>
> Time Spent: 1h 20m
> Remaining Estimate: 0h
>
> For debugging query issues, basic details like table schema, statistics,
> partition details, query plans are needed.
> It would be good to have "explain ddl" support, which can generate these
> details. This can help in recreating the schema and planner issues without
> sample data.
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
