http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java index 4e3068d..f2642cf 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionTxnHandler.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.metastore.txn; +import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.classification.RetrySemantics; import org.apache.hadoop.hive.metastore.api.CompactionType; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -576,8 +577,8 @@ class CompactionTxnHandler extends TxnHandler { dbConn = getDbConn(Connection.TRANSACTION_READ_COMMITTED); stmt = dbConn.createStatement(); String s = "select txn_id from TXNS where " + - "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " + - "txn_state = '" + TXN_ABORTED + "'"; + "txn_id not in (select tc_txnid from TXN_COMPONENTS) and " + + "txn_state = '" + TXN_ABORTED + "'"; LOG.debug("Going to execute query <" + s + ">"); rs = stmt.executeQuery(s); List<Long> txnids = new ArrayList<>(); @@ -587,10 +588,71 @@ class CompactionTxnHandler extends TxnHandler { return; } Collections.sort(txnids);//easier to read logs + List<String> queries = new ArrayList<>(); StringBuilder prefix = new StringBuilder(); StringBuilder suffix = new StringBuilder(); + // Turn off COLUMN_STATS_ACCURATE for txnids' components in TBLS and PARTITIONS + for (Long txnId : txnids) { + // Get table ids for the current txnId. + s = "select tbl_id from TBLS where txn_id = " + txnId; + LOG.debug("Going to execute query <" + s + ">"); + rs = stmt.executeQuery(s); + List<Long> tblIds = new ArrayList<>(); + while (rs.next()) { + tblIds.add(rs.getLong(1)); + } + close(rs); + if(tblIds.size() <= 0) { + continue; + } + + // Update COLUMN_STATS_AcCURATE.BASIC_STATS to false for each tableId. + prefix.append("delete from TABLE_PARAMS " + + " where param_key = '" + "COLUMN_STATS_ACCURATE" + "' and "); + suffix.append(""); + TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, tblIds, "tbl_id", true, false); + + for (String query : queries) { + LOG.debug("Going to execute update <" + query + ">"); + int rc = stmt.executeUpdate(query); + LOG.info("Turned off " + rc + " COLUMN_STATE_ACCURATE.BASIC_STATS states from TBLS"); + } + + queries.clear(); + prefix.setLength(0); + suffix.setLength(0); + + // Get partition ids for the current txnId. + s = "select part_id from PARTITIONS where txn_id = " + txnId; + LOG.debug("Going to execute query <" + s + ">"); + rs = stmt.executeQuery(s); + List<Long> ptnIds = new ArrayList<>(); + while (rs.next()) ptnIds.add(rs.getLong(1)); + close(rs); + if(ptnIds.size() <= 0) { + continue; + } + + // Update COLUMN_STATS_AcCURATE.BASIC_STATS to false for each ptnId. + prefix.append("delete from PARTITION_PARAMS " + + " where param_key = '" + "COLUMN_STATS_ACCURATE" + "' and "); + suffix.append(""); + TxnUtils.buildQueryWithINClause(conf, queries, prefix, suffix, ptnIds, "part_id", true, false); + + for (String query : queries) { + LOG.debug("Going to execute update <" + query + ">"); + int rc = stmt.executeUpdate(query); + LOG.info("Turned off " + rc + " COLUMN_STATE_ACCURATE.BASIC_STATS states from PARTITIONS"); + } + + queries.clear(); + prefix.setLength(0); + suffix.setLength(0); + } + + // Delete from TXNS. prefix.append("delete from TXNS where "); suffix.append("");
http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index 50bfca3..bfbd928 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -28,9 +28,12 @@ import java.sql.Statement; import java.util.Properties; import com.google.common.annotations.VisibleForTesting; +import jline.internal.Log; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; +import org.apache.zookeeper.txn.TxnHeader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -195,6 +198,68 @@ public final class TxnDbUtil { ); try { + stmt.execute("CREATE TABLE \"APP\".\"TBLS\" (\"TBL_ID\" BIGINT NOT NULL, " + + " \"CREATE_TIME\" INTEGER NOT NULL, \"DB_ID\" BIGINT, \"LAST_ACCESS_TIME\" INTEGER NOT NULL, " + + " \"OWNER\" VARCHAR(767), \"OWNER_TYPE\" VARCHAR(10), \"RETENTION\" INTEGER NOT NULL, " + + " \"SD_ID\" BIGINT, \"TBL_NAME\" VARCHAR(256), \"TBL_TYPE\" VARCHAR(128), " + + " \"VIEW_EXPANDED_TEXT\" LONG VARCHAR, \"VIEW_ORIGINAL_TEXT\" LONG VARCHAR, " + + " \"IS_REWRITE_ENABLED\" CHAR(1) NOT NULL DEFAULT \'N\', \"TXN_ID\" BIGINT DEFAULT 0, " + + " \"WRITEID_LIST\" CLOB, " + + " PRIMARY KEY (TBL_ID))" + ); + } catch (SQLException e) { + if (e.getMessage() != null && e.getMessage().contains("already exists")) { + LOG.info("TBLS table already exist, ignoring"); + } else { + throw e; + } + } + + try { + stmt.execute("CREATE TABLE \"APP\".\"PARTITIONS\" (" + + " \"PART_ID\" BIGINT NOT NULL, \"CREATE_TIME\" INTEGER NOT NULL, " + + " \"LAST_ACCESS_TIME\" INTEGER NOT NULL, \"PART_NAME\" VARCHAR(767), " + + " \"SD_ID\" BIGINT, \"TBL_ID\" BIGINT, \"TXN_ID\" BIGINT DEFAULT 0, " + + " \"WRITEID_LIST\" CLOB, " + + " PRIMARY KEY (PART_ID))" + ); + } catch (SQLException e) { + if (e.getMessage() != null && e.getMessage().contains("already exists")) { + LOG.info("PARTITIONS table already exist, ignoring"); + } else { + throw e; + } + } + + try { + stmt.execute("CREATE TABLE \"APP\".\"TABLE_PARAMS\" (" + + " \"TBL_ID\" BIGINT NOT NULL, \"PARAM_KEY\" VARCHAR(256) NOT NULL, " + + " \"PARAM_VALUE\" CLOB, " + + " PRIMARY KEY (TBL_ID, PARAM_KEY))" + ); + } catch (SQLException e) { + if (e.getMessage() != null && e.getMessage().contains("already exists")) { + LOG.info("TABLE_PARAMS table already exist, ignoring"); + } else { + throw e; + } + } + + try { + stmt.execute("CREATE TABLE \"APP\".\"PARTITION_PARAMS\" (" + + " \"PART_ID\" BIGINT NOT NULL, \"PARAM_KEY\" VARCHAR(256) NOT NULL, " + + " \"PARAM_VALUE\" VARCHAR(4000), " + + " PRIMARY KEY (PART_ID, PARAM_KEY))" + ); + } catch (SQLException e) { + if (e.getMessage() != null && e.getMessage().contains("already exists")) { + LOG.info("PARTITION_PARAMS table already exist, ignoring"); + } else { + throw e; + } + } + + try { stmt.execute("CREATE TABLE \"APP\".\"SEQUENCE_TABLE\" (\"SEQUENCE_NAME\" VARCHAR(256) NOT " + "NULL, \"NEXT_VAL\" BIGINT NOT NULL)" @@ -376,6 +441,35 @@ public final class TxnDbUtil { } /** + * Return true if the transaction of the given txnId is open. + * @param conf HiveConf + * @param txnId transaction id to search for + * @return + * @throws Exception + */ + public static boolean isOpenOrAbortedTransaction(Configuration conf, long txnId) throws Exception { + Connection conn = null; + PreparedStatement stmt = null; + ResultSet rs = null; + try { + conn = getConnection(conf); + conn.setAutoCommit(false); + conn.setTransactionIsolation(Connection.TRANSACTION_READ_COMMITTED); + + stmt = conn.prepareStatement("SELECT txn_id FROM TXNS WHERE txn_id = ?"); + stmt.setLong(1, txnId); + rs = stmt.executeQuery(); + if (!rs.next()) { + return false; + } else { + return true; + } + } finally { + closeResources(conn, stmt, rs); + } + } + + /** * Utility method used to run COUNT queries like "select count(*) from ..." against metastore tables * @param countQuery countQuery text * @return count countQuery result http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java index fa291d5..aac5811 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnUtils.java @@ -25,11 +25,7 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.common.ValidTxnWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.metastore.TransactionalValidationListener; -import org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse; -import org.apache.hadoop.hive.metastore.api.GetValidWriteIdsResponse; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; -import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; import org.apache.hadoop.hive.metastore.utils.JavaUtils; @@ -46,6 +42,12 @@ import java.util.Map; public class TxnUtils { private static final Logger LOG = LoggerFactory.getLogger(TxnUtils.class); + // Transactional stats states + static final public char STAT_OPEN = 'o'; + static final public char STAT_INVALID = 'i'; + static final public char STAT_COMMITTED = 'c'; + static final public char STAT_OBSOLETE = 's'; + /** * Transform a {@link org.apache.hadoop.hive.metastore.api.GetOpenTxnsResponse} to a * {@link org.apache.hadoop.hive.common.ValidTxnList}. This assumes that the caller intends to @@ -223,6 +225,14 @@ public class TxnUtils { return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true"); } + public static boolean isTransactionalTable(Map<String, String> parameters) { + if (parameters == null) { + return false; + } + String tableIsTransactional = parameters.get(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL); + return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true"); + } + /** * Should produce the same result as * {@link org.apache.hadoop.hive.ql.io.AcidUtils#isAcidTable(org.apache.hadoop.hive.ql.metadata.Table)}. http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/resources/package.jdo ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/resources/package.jdo b/standalone-metastore/src/main/resources/package.jdo index 1be3e98..3997f53 100644 --- a/standalone-metastore/src/main/resources/package.jdo +++ b/standalone-metastore/src/main/resources/package.jdo @@ -210,6 +210,12 @@ <field name="tableType"> <column name="TBL_TYPE" length="128" jdbc-type="VARCHAR"/> </field> + <field name="txnId"> + <column name="TXN_ID"/> + </field> + <field name="writeIdList"> + <column name="WRITEID_LIST" jdbc-type="CLOB" allows-null="true"/> + </field> </class> <class name="MCreationMetadata" identity-type="datastore" table="MV_CREATION_METADATA" detachable="true"> @@ -489,6 +495,12 @@ <column name="PARAM_VALUE" length="4000" jdbc-type="VARCHAR"/> </value> </field> + <field name="txnId"> + <column name="TXN_ID"/> + </field> + <field name="writeIdList"> + <column name="WRITEID_LIST" jdbc-type="CLOB" allows-null="true"/> + </field> </class> <class name="MIndex" table="IDXS" identity-type="datastore" detachable="true"> @@ -989,6 +1001,9 @@ <field name="lastAnalyzed"> <column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/> </field> + <field name="txnId"> + <column name="TXN_ID"/> + </field> </class> <class name="MPartitionColumnStatistics" table="PART_COL_STATS" identity-type="datastore" detachable="true"> @@ -1059,6 +1074,9 @@ <field name="lastAnalyzed"> <column name="LAST_ANALYZED" jdbc-type="BIGINT" allows-null="false"/> </field> + <field name="txnId"> + <column name="TXN_ID"/> + </field> </class> <class name="MVersionTable" table="VERSION" identity-type="datastore" detachable="true"> <datastore-identity> http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index e818e1b..280fd4a 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -47,7 +47,7 @@ CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); -CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT); +CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB); CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS" VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER); @@ -75,7 +75,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), " CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128)); -CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N'); +CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB); CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); @@ -106,7 +106,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"( "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, - "BIT_VECTOR" BLOB + "BIT_VECTOR" BLOB, + "TXN_ID" BIGINT DEFAULT 0 ); CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); @@ -155,7 +156,8 @@ CREATE TABLE "APP"."PART_COL_STATS"( "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, - "PART_ID" BIGINT NOT NULL + "PART_ID" BIGINT NOT NULL, + "TXN_ID" BIGINT DEFAULT 0 ); CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255)); @@ -373,7 +375,6 @@ ALTER TABLE "APP"."MV_CREATION_METADATA" ADD CONSTRAINT "MV_CREATION_METADATA_PK ALTER TABLE "APP"."CTLGS" ADD CONSTRAINT "CTLG_PK" PRIMARY KEY ("CTLG_ID"); - -- foreign ALTER TABLE "APP"."IDXS" ADD CONSTRAINT "IDXS_FK1" FOREIGN KEY ("ORIG_TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql index bb69105..f92f13c 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-4.0.0.derby.sql @@ -47,7 +47,7 @@ CREATE TABLE "APP"."IDXS" ("INDEX_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT CREATE TABLE "APP"."INDEX_PARAMS" ("INDEX_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" VARCHAR(4000)); -CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT); +CREATE TABLE "APP"."PARTITIONS" ("PART_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "LAST_ACCESS_TIME" INTEGER NOT NULL, "PART_NAME" VARCHAR(767), "SD_ID" BIGINT, "TBL_ID" BIGINT, "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB); CREATE TABLE "APP"."SERDES" ("SERDE_ID" BIGINT NOT NULL, "NAME" VARCHAR(128), "SLIB" VARCHAR(4000), "DESCRIPTION" VARCHAR(4000), "SERIALIZER_CLASS" VARCHAR(4000), "DESERIALIZER_CLASS" VARCHAR(4000), SERDE_TYPE INTEGER); @@ -75,7 +75,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), " CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128)); -CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N'); +CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "OWNER_TYPE" VARCHAR(10), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(256), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "IS_REWRITE_ENABLED" CHAR(1) NOT NULL DEFAULT 'N', "TXN_ID" BIGINT DEFAULT 0, "WRITEID_LIST" CLOB); CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); @@ -106,7 +106,8 @@ CREATE TABLE "APP"."TAB_COL_STATS"( "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, - "BIT_VECTOR" BLOB + "BIT_VECTOR" BLOB, + "TXN_ID" BIGINT DEFAULT 0 ); CREATE TABLE "APP"."TABLE_PARAMS" ("TBL_ID" BIGINT NOT NULL, "PARAM_KEY" VARCHAR(256) NOT NULL, "PARAM_VALUE" CLOB); @@ -155,7 +156,8 @@ CREATE TABLE "APP"."PART_COL_STATS"( "NUM_FALSES" BIGINT, "LAST_ANALYZED" BIGINT, "CS_ID" BIGINT NOT NULL, - "PART_ID" BIGINT NOT NULL + "PART_ID" BIGINT NOT NULL, + "TXN_ID" BIGINT DEFAULT 0 ); CREATE TABLE "APP"."VERSION" ("VER_ID" BIGINT NOT NULL, "SCHEMA_VERSION" VARCHAR(127) NOT NULL, "VERSION_COMMENT" VARCHAR(255)); http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql index a511376..94f8192 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-3.1.0-to-4.0.0.derby.sql @@ -1,5 +1,11 @@ -- Upgrade MetaStore schema from 3.1.0 to 4.0.0 - +-- HIVE-19416 +ALTER TABLE "APP"."TBLS" ADD WRITEID_LIST CLOB; +ALTER TABLE "APP"."TBLS" ADD TXN_ID bigint DEFAULT 0; +ALTER TABLE "APP"."PARTITIONS" ADD WRITEID_LIST CLOB; +ALTER TABLE "APP"."PARTITIONS" ADD TXN_ID bigint DEFAULT 0; +ALTER TABLE "APP"."TAB_COL_STATS" ADD TXN_ID bigint DEFAULT 0; +ALTER TABLE "APP"."PART_COL_STATS" ADD TXN_ID bigint DEFAULT 0; -- This needs to be the last thing done. Insert any changes above this line. UPDATE "APP".VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index c88fb18..f20f910 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -94,7 +94,8 @@ CREATE TABLE PART_COL_STATS PART_ID bigint NULL, PARTITION_NAME nvarchar(767) NOT NULL, "TABLE_NAME" nvarchar(256) NOT NULL, - "CAT_NAME" nvarchar(256) NOT NULL + "CAT_NAME" nvarchar(256) NOT NULL, + TXN_ID bigint NULL ); ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PK PRIMARY KEY (CS_ID); @@ -144,7 +145,9 @@ CREATE TABLE PARTITIONS LAST_ACCESS_TIME int NOT NULL, PART_NAME nvarchar(767) NULL, SD_ID bigint NULL, - TBL_ID bigint NULL + TBL_ID bigint NULL, + TXN_ID bigint NULL, + WRITEID_LIST text NULL ); ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); @@ -238,7 +241,8 @@ CREATE TABLE TAB_COL_STATS NUM_TRUES bigint NULL, TBL_ID bigint NULL, "TABLE_NAME" nvarchar(256) NOT NULL, - "CAT_NAME" nvarchar(256) NOT NULL + "CAT_NAME" nvarchar(256) NOT NULL, + TXN_ID bigint NULL ); ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID); @@ -369,7 +373,9 @@ CREATE TABLE TBLS TBL_TYPE nvarchar(128) NULL, VIEW_EXPANDED_TEXT text NULL, VIEW_ORIGINAL_TEXT text NULL, - IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0 + IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0, + TXN_ID bigint NULL, + WRITEID_LIST text NULL ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql index 922e8fe..22637c5 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-4.0.0.mssql.sql @@ -94,7 +94,8 @@ CREATE TABLE PART_COL_STATS PART_ID bigint NULL, PARTITION_NAME nvarchar(767) NOT NULL, "TABLE_NAME" nvarchar(256) NOT NULL, - "CAT_NAME" nvarchar(256) NOT NULL + "CAT_NAME" nvarchar(256) NOT NULL, + TXN_ID bigint NULL ); ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PK PRIMARY KEY (CS_ID); @@ -145,7 +146,9 @@ CREATE TABLE PARTITIONS LAST_ACCESS_TIME int NOT NULL, PART_NAME nvarchar(767) NULL, SD_ID bigint NULL, - TBL_ID bigint NULL + TBL_ID bigint NULL, + TXN_ID bigint NULL, + WRITEID_LIST text NULL ); ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); @@ -242,7 +245,8 @@ CREATE TABLE TAB_COL_STATS NUM_TRUES bigint NULL, TBL_ID bigint NULL, "TABLE_NAME" nvarchar(256) NOT NULL, - "CAT_NAME" nvarchar(256) NOT NULL + "CAT_NAME" nvarchar(256) NOT NULL, + TXN_ID bigint NULL ); ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PK PRIMARY KEY (CS_ID); @@ -377,7 +381,9 @@ CREATE TABLE TBLS TBL_TYPE nvarchar(128) NULL, VIEW_EXPANDED_TEXT text NULL, VIEW_ORIGINAL_TEXT text NULL, - IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0 + IS_REWRITE_ENABLED bit NOT NULL DEFAULT 0, + TXN_ID bigint NULL, + WRITEID_LIST text NULL ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql index 27b7026..f0cbf6c 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-3.1.0-to-4.0.0.mssql.sql @@ -1,5 +1,13 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS MESSAGE; +-- HIVE-19416 +ALTER TABLE TBLS ADD WRITEID_LIST text NULL; +ALTER TABLE TBLS ADD TXN_ID bigint NULL; +ALTER TABLE PARTITIONS ADD WRITEID_LIST text NULL; +ALTER TABLE PARTITIONS ADD TXN_ID bigint NULL; +ALTER TABLE TAB_COL_STATS ADD TXN_ID bigint NULL; +ALTER TABLE PART_COL_STATS ADD TXN_ID bigint NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index c54df55..6e34ab5 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -222,6 +222,8 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` ( `PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `SD_ID` bigint(20) DEFAULT NULL, `TBL_ID` bigint(20) DEFAULT NULL, + `TXN_ID` bigint(20) DEFAULT 0, + `WRITEID_LIST` text DEFAULT NULL, PRIMARY KEY (`PART_ID`), UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`), KEY `PARTITIONS_N49` (`TBL_ID`), @@ -625,6 +627,8 @@ CREATE TABLE IF NOT EXISTS `TBLS` ( `VIEW_EXPANDED_TEXT` mediumtext, `VIEW_ORIGINAL_TEXT` mediumtext, `IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0, + `TXN_ID` bigint(20) DEFAULT 0, + `WRITEID_LIST` text DEFAULT NULL, PRIMARY KEY (`TBL_ID`), UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), KEY `TBLS_N50` (`SD_ID`), @@ -720,6 +724,7 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( `NUM_TRUES` bigint(20), `NUM_FALSES` bigint(20), `LAST_ANALYZED` bigint(20) NOT NULL, + `TXN_ID` bigint(20) DEFAULT 0, PRIMARY KEY (`CS_ID`), CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -750,6 +755,7 @@ CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( `NUM_TRUES` bigint(20), `NUM_FALSES` bigint(20), `LAST_ANALYZED` bigint(20) NOT NULL, + `TXN_ID` bigint(20) DEFAULT 0, PRIMARY KEY (`CS_ID`), CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql index 6c40e6e..f8f229d 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-4.0.0.mysql.sql @@ -224,6 +224,8 @@ CREATE TABLE IF NOT EXISTS `PARTITIONS` ( `PART_NAME` varchar(767) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `SD_ID` bigint(20) DEFAULT NULL, `TBL_ID` bigint(20) DEFAULT NULL, + `TXN_ID` bigint(20) DEFAULT 0, + `WRITEID_LIST` text DEFAULT NULL, PRIMARY KEY (`PART_ID`), UNIQUE KEY `UNIQUEPARTITION` (`PART_NAME`,`TBL_ID`), KEY `PARTITIONS_N49` (`TBL_ID`), @@ -629,6 +631,8 @@ CREATE TABLE IF NOT EXISTS `TBLS` ( `VIEW_EXPANDED_TEXT` mediumtext, `VIEW_ORIGINAL_TEXT` mediumtext, `IS_REWRITE_ENABLED` bit(1) NOT NULL DEFAULT 0, + `TXN_ID` bigint(20) DEFAULT 0, + `WRITEID_LIST` text DEFAULT NULL, PRIMARY KEY (`TBL_ID`), UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), KEY `TBLS_N50` (`SD_ID`), @@ -726,6 +730,7 @@ CREATE TABLE IF NOT EXISTS `TAB_COL_STATS` ( `NUM_TRUES` bigint(20), `NUM_FALSES` bigint(20), `LAST_ANALYZED` bigint(20) NOT NULL, + `TXN_ID` bigint(20) DEFAULT 0, PRIMARY KEY (`CS_ID`), CONSTRAINT `TAB_COL_STATS_FK` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; @@ -757,6 +762,7 @@ CREATE TABLE IF NOT EXISTS `PART_COL_STATS` ( `NUM_TRUES` bigint(20), `NUM_FALSES` bigint(20), `LAST_ANALYZED` bigint(20) NOT NULL, + `TXN_ID` bigint(20) DEFAULT 0, PRIMARY KEY (`CS_ID`), CONSTRAINT `PART_COL_STATS_FK` FOREIGN KEY (`PART_ID`) REFERENCES `PARTITIONS` (`PART_ID`) ) ENGINE=InnoDB DEFAULT CHARSET=latin1; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index 9b87563..5877c93 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -323,4 +323,4 @@ ALTER TABLE TXN_COMPONENTS MODIFY COLUMN TC_TABLE varchar(128) NULL; UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS ' '; -ALTER TABLE `TBLS` ADD COLUMN `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL; \ No newline at end of file +ALTER TABLE `TBLS` ADD COLUMN `OWNER_TYPE` VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql index b3789f9..4ca584c 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-3.1.0-to-4.0.0.mysql.sql @@ -1,5 +1,13 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS ' '; +-- HIVE-19416 +ALTER TABLE TBLS ADD TXN_ID bigint; +ALTER TABLE TBLS ADD WRITEID_LIST CLOB; +ALTER TABLE PARTITIONS ADD TXN_ID bigint; +ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB; +ALTER TABLE TAB_COL_STATS ADD TXN_ID bigint; +ALTER TABLE PART_COL_STATS ADD TXN_ID bigint; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0' AS ' '; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index 63cc1f7..abdb984 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -162,7 +162,9 @@ CREATE TABLE PARTITIONS LAST_ACCESS_TIME NUMBER (10) NOT NULL, PART_NAME VARCHAR2(767) NULL, SD_ID NUMBER NULL, - TBL_ID NUMBER NULL + TBL_ID NUMBER NULL, + TXN_ID NUMBER NULL, + WRITEID_LIST CLOB NULL ); ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); @@ -392,7 +394,9 @@ CREATE TABLE TBLS TBL_TYPE VARCHAR2(128) NULL, VIEW_EXPANDED_TEXT CLOB NULL, VIEW_ORIGINAL_TEXT CLOB NULL, - IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)) + IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)), + TXN_ID NUMBER NULL, + WRITEID_LIST CLOB NULL ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); @@ -525,7 +529,8 @@ CREATE TABLE TAB_COL_STATS ( MAX_COL_LEN NUMBER, NUM_TRUES NUMBER, NUM_FALSES NUMBER, - LAST_ANALYZED NUMBER NOT NULL + LAST_ANALYZED NUMBER NOT NULL, + TXN_ID NUMBER NULL ); CREATE TABLE VERSION ( @@ -563,7 +568,8 @@ CREATE TABLE PART_COL_STATS ( MAX_COL_LEN NUMBER, NUM_TRUES NUMBER, NUM_FALSES NUMBER, - LAST_ANALYZED NUMBER NOT NULL + LAST_ANALYZED NUMBER NOT NULL, + TXN_ID NUMBER NULL ); ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PKEY PRIMARY KEY (CS_ID); @@ -1134,7 +1140,6 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql index e12150a..a143fd2 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-4.0.0.oracle.sql @@ -163,7 +163,9 @@ CREATE TABLE PARTITIONS LAST_ACCESS_TIME NUMBER (10) NOT NULL, PART_NAME VARCHAR2(767) NULL, SD_ID NUMBER NULL, - TBL_ID NUMBER NULL + TBL_ID NUMBER NULL, + TXN_ID NUMBER NULL, + WRITEID_LIST CLOB NULL ); ALTER TABLE PARTITIONS ADD CONSTRAINT PARTITIONS_PK PRIMARY KEY (PART_ID); @@ -398,7 +400,9 @@ CREATE TABLE TBLS TBL_TYPE VARCHAR2(128) NULL, VIEW_EXPANDED_TEXT CLOB NULL, VIEW_ORIGINAL_TEXT CLOB NULL, - IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)) + IS_REWRITE_ENABLED NUMBER(1) DEFAULT 0 NOT NULL CHECK (IS_REWRITE_ENABLED IN (1,0)), + TXN_ID NUMBER NULL, + WRITEID_LIST CLOB NULL ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); @@ -531,7 +535,8 @@ CREATE TABLE TAB_COL_STATS ( MAX_COL_LEN NUMBER, NUM_TRUES NUMBER, NUM_FALSES NUMBER, - LAST_ANALYZED NUMBER NOT NULL + LAST_ANALYZED NUMBER NOT NULL, + TXN_ID NUMBER NULL ); ALTER TABLE TAB_COL_STATS ADD CONSTRAINT TAB_COL_STATS_PKEY PRIMARY KEY (CS_ID); @@ -571,7 +576,8 @@ CREATE TABLE PART_COL_STATS ( MAX_COL_LEN NUMBER, NUM_TRUES NUMBER, NUM_FALSES NUMBER, - LAST_ANALYZED NUMBER NOT NULL + LAST_ANALYZED NUMBER NOT NULL, + TXN_ID NUMBER NULL ); ALTER TABLE PART_COL_STATS ADD CONSTRAINT PART_COL_STATS_PKEY PRIMARY KEY (CS_ID); http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index ce3437f..5b767bc 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -339,4 +339,4 @@ UPDATE COMPLETED_TXN_COMPONENTS SET CTC_WRITEID = CTC_TXNID; UPDATE VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.3.0 to 3.0.0' AS Status from dual; -ALTER TABLE TBLS ADD OWNER_TYPE VARCHAR2(10) NULL; \ No newline at end of file +ALTER TABLE TBLS ADD OWNER_TYPE VARCHAR2(10) NULL; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql index 6fa5e2d..7ac4d40 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-3.1.0-to-4.0.0.oracle.sql @@ -1,5 +1,12 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0' AS Status from dual; +ALTER TABLE TBLS ADD TXN_ID number NULL; +ALTER TABLE TBLS ADD WRITEID_LIST CLOB NULL; +ALTER TABLE PARTITIONS ADD TXN_ID number NULL; +ALTER TABLE PARTITIONS ADD WRITEID_LIST CLOB NULL; +ALTER TABLE TAB_COL_STATS ADD TXN_ID number NULL; +ALTER TABLE PART_COL_STATS ADD TXN_ID number NULL; + -- These lines need to be last. Insert any changes above. UPDATE VERSION SET SCHEMA_VERSION='4.0.0', VERSION_COMMENT='Hive release version 4.0.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0' AS Status from dual; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index 97697f8..449f295 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -166,7 +166,9 @@ CREATE TABLE "PARTITIONS" ( "LAST_ACCESS_TIME" bigint NOT NULL, "PART_NAME" character varying(767) DEFAULT NULL::character varying, "SD_ID" bigint, - "TBL_ID" bigint + "TBL_ID" bigint, + "TXN_ID" bigint, + "WRITEID_LIST" text ); @@ -388,7 +390,9 @@ CREATE TABLE "TBLS" ( "TBL_TYPE" character varying(128) DEFAULT NULL::character varying, "VIEW_EXPANDED_TEXT" text, "VIEW_ORIGINAL_TEXT" text, - "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false + "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false, + "TXN_ID" bigint, + "WRITEID_LIST" text ); -- @@ -539,7 +543,8 @@ CREATE TABLE "TAB_COL_STATS" ( "MAX_COL_LEN" bigint, "NUM_TRUES" bigint, "NUM_FALSES" bigint, - "LAST_ANALYZED" bigint NOT NULL + "LAST_ANALYZED" bigint NOT NULL, + "TXN_ID" bigint ); -- @@ -577,7 +582,8 @@ CREATE TABLE "PART_COL_STATS" ( "MAX_COL_LEN" bigint, "NUM_TRUES" bigint, "NUM_FALSES" bigint, - "LAST_ANALYZED" bigint NOT NULL + "LAST_ANALYZED" bigint NOT NULL, + "TXN_ID" bigint ); -- @@ -1074,6 +1080,8 @@ ALTER TABLE ONLY "WM_MAPPING" ALTER TABLE ONLY "WM_MAPPING" ADD CONSTRAINT "UNIQUE_WM_MAPPING" UNIQUE ("RP_ID", "ENTITY_TYPE", "ENTITY_NAME"); +-- Transactional table stats PK constraints + -- -- Name: BUCKETING_COLS_N49; Type: INDEX; Schema: public; Owner: hiveuser; Tablespace: -- @@ -1618,6 +1626,8 @@ ALTER TABLE ONLY "MV_TABLES_USED" ALTER TABLE ONLY "MV_TABLES_USED" ADD CONSTRAINT "MV_TABLES_USED_FK2" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS" ("TBL_ID") DEFERRABLE; +-- Transactional table stats FK constraints + -- -- Name: public; Type: ACL; Schema: -; Owner: hiveuser -- @@ -1822,7 +1832,6 @@ CREATE TABLE RUNTIME_STATS ( CREATE INDEX IDX_RUNTIME_STATS_CREATE_TIME ON RUNTIME_STATS(CREATE_TIME); - -- ----------------------------------------------------------------- -- Record schema version. Should be the last step in the init script -- ----------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql index b73e1d1..0ead590 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-4.0.0.postgres.sql @@ -168,7 +168,9 @@ CREATE TABLE "PARTITIONS" ( "LAST_ACCESS_TIME" bigint NOT NULL, "PART_NAME" character varying(767) DEFAULT NULL::character varying, "SD_ID" bigint, - "TBL_ID" bigint + "TBL_ID" bigint, + "TXN_ID" bigint, + "WRITEID_LIST" text ); @@ -392,7 +394,9 @@ CREATE TABLE "TBLS" ( "TBL_TYPE" character varying(128) DEFAULT NULL::character varying, "VIEW_EXPANDED_TEXT" text, "VIEW_ORIGINAL_TEXT" text, - "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false + "IS_REWRITE_ENABLED" boolean NOT NULL DEFAULT false, + "TXN_ID" bigint, + "WRITEID_LIST" text ); -- @@ -545,7 +549,8 @@ CREATE TABLE "TAB_COL_STATS" ( "MAX_COL_LEN" bigint, "NUM_TRUES" bigint, "NUM_FALSES" bigint, - "LAST_ANALYZED" bigint NOT NULL + "LAST_ANALYZED" bigint NOT NULL, + "TXN_ID" bigint ); -- @@ -583,7 +588,8 @@ CREATE TABLE "PART_COL_STATS" ( "MAX_COL_LEN" bigint, "NUM_TRUES" bigint, "NUM_FALSES" bigint, - "LAST_ANALYZED" bigint NOT NULL + "LAST_ANALYZED" bigint NOT NULL, + "TXN_ID" bigint ); -- http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql index 40d2795..f2bae02 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-3.1.0-to-4.0.0.postgres.sql @@ -1,5 +1,13 @@ SELECT 'Upgrading MetaStore schema from 3.1.0 to 4.0.0'; +-- HIVE-19416 +ALTER TABLE "TBLS" ADD "TXN_ID" bigint; +ALTER TABLE "TBLS" ADD "WRITEID_LIST" text; +ALTER TABLE "PARTITIONS" ADD "TXN_ID" bigint; +ALTER TABLE "PARTITIONS" ADD "WRITEID_LIST" text; +ALTER TABLE "TAB_COL_STATS" ADD "TXN_ID" bigint; +ALTER TABLE "PART_COL_STATS" ADD "TXN_ID" bigint; + -- These lines need to be last. Insert any changes above. UPDATE "VERSION" SET "SCHEMA_VERSION"='4.0.0', "VERSION_COMMENT"='Hive release version 4.0.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 3.1.0 to 4.0.0'; http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index 6e503eb..ecd2001 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -233,6 +233,12 @@ enum SchemaVersionState { DELETED = 8 } +enum IsolationLevelCompliance { + YES = 1, + NO = 2, + UNKNOWN = 3 +} + struct HiveObjectRef{ 1: HiveObjectType objectType, 2: string dbName, @@ -430,7 +436,10 @@ struct Table { 15: optional bool rewriteEnabled, // rewrite enabled or not 16: optional CreationMetadata creationMetadata, // only for MVs, it stores table names used and txn list at MV creation 17: optional string catName, // Name of the catalog the table is in - 18: optional PrincipalType ownerType = PrincipalType.USER // owner type of this table (default to USER for backward compatibility) + 18: optional PrincipalType ownerType = PrincipalType.USER, // owner type of this table (default to USER for backward compatibility) + 19: optional i64 txnId=-1, + 20: optional string validWriteIdList, + 21: optional IsolationLevelCompliance isStatsCompliant } struct Partition { @@ -442,7 +451,10 @@ struct Partition { 6: StorageDescriptor sd, 7: map<string, string> parameters, 8: optional PrincipalPrivilegeSet privileges, - 9: optional string catName + 9: optional string catName, + 10: optional i64 txnId=-1, + 11: optional string validWriteIdList, + 12: optional IsolationLevelCompliance isStatsCompliant } struct PartitionWithoutSD { @@ -469,7 +481,10 @@ struct PartitionSpec { 3: string rootPath, 4: optional PartitionSpecWithSharedSD sharedSDPartitionSpec, 5: optional PartitionListComposingSpec partitionList, - 6: optional string catName + 6: optional string catName, + 7: optional i64 txnId=-1, + 8: optional string validWriteIdList, + 9: optional IsolationLevelCompliance isStatsCompliant } // column statistics @@ -564,17 +579,24 @@ struct ColumnStatisticsDesc { struct ColumnStatistics { 1: required ColumnStatisticsDesc statsDesc, -2: required list<ColumnStatisticsObj> statsObj; +2: required list<ColumnStatisticsObj> statsObj, +3: optional i64 txnId=-1, // transaction id of the query that sends this structure +4: optional string validWriteIdList, // valid write id list for the table for which this struct is being sent +5: optional IsolationLevelCompliance isStatsCompliant // Are the stats isolation-level-compliant with the + // the calling query? } struct AggrStats { 1: required list<ColumnStatisticsObj> colStats, -2: required i64 partsFound // number of partitions for which stats were found +2: required i64 partsFound, // number of partitions for which stats were found +3: optional IsolationLevelCompliance isStatsCompliant } struct SetPartitionsStatsRequest { 1: required list<ColumnStatistics> colStats, -2: optional bool needMerge //stats need to be merged with the existing stats +2: optional bool needMerge, //stats need to be merged with the existing stats +3: optional i64 txnId=-1, // transaction id of the query that sends this structure +4: optional string validWriteIdList // valid write id list for the table for which this struct is being sent } // schema of the table/query results etc. @@ -703,18 +725,22 @@ struct PartitionsByExprRequest { } struct TableStatsResult { - 1: required list<ColumnStatisticsObj> tableStats + 1: required list<ColumnStatisticsObj> tableStats, + 2: optional IsolationLevelCompliance isStatsCompliant } struct PartitionsStatsResult { - 1: required map<string, list<ColumnStatisticsObj>> partStats + 1: required map<string, list<ColumnStatisticsObj>> partStats, + 2: optional IsolationLevelCompliance isStatsCompliant } struct TableStatsRequest { 1: required string dbName, 2: required string tblName, 3: required list<string> colNames - 4: optional string catName + 4: optional string catName, + 5: optional i64 txnId=-1, // transaction id of the query that sends this structure + 6: optional string validWriteIdList // valid write id list for the table for which this struct is being sent } struct PartitionsStatsRequest { @@ -722,12 +748,15 @@ struct PartitionsStatsRequest { 2: required string tblName, 3: required list<string> colNames, 4: required list<string> partNames, - 5: optional string catName + 5: optional string catName, + 6: optional i64 txnId=-1, // transaction id of the query that sends this structure + 7: optional string validWriteIdList // valid write id list for the table for which this struct is being sent } // Return type for add_partitions_req struct AddPartitionsResult { 1: optional list<Partition> partitions, + 2: optional IsolationLevelCompliance isStatsCompliant } // Request type for add_partitions_req @@ -737,7 +766,9 @@ struct AddPartitionsRequest { 3: required list<Partition> parts, 4: required bool ifNotExists, 5: optional bool needResult=true, - 6: optional string catName + 6: optional string catName, + 7: optional i64 txnId=-1, + 8: optional string validWriteIdList } // Return type for drop_partitions_req @@ -1209,11 +1240,14 @@ struct GetTableRequest { 1: required string dbName, 2: required string tblName, 3: optional ClientCapabilities capabilities, - 4: optional string catName + 4: optional string catName, + 5: optional i64 txnId=-1, + 6: optional string validWriteIdList } struct GetTableResult { - 1: required Table table + 1: required Table table, + 2: optional IsolationLevelCompliance isStatsCompliant } struct GetTablesRequest { @@ -1544,6 +1578,18 @@ struct GetRuntimeStatsRequest { 2: required i32 maxCreateTime } +struct AlterPartitionsRequest { + 1: required string dbName, + 2: required string tableName, + 3: required list<Partition> partitions, + 4: required EnvironmentContext environmentContext, + 5: optional i64 txnId=-1, + 6: optional string validWriteIdList +} + +struct AlterPartitionsResponse { +} + // Exceptions. exception MetaException { @@ -1874,7 +1920,9 @@ service ThriftHiveMetastore extends fb303.FacebookService // prehooks are fired together followed by all post hooks void alter_partitions(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts) throws (1:InvalidOperationException o1, 2:MetaException o2) - void alter_partitions_with_environment_context(1:string db_name, 2:string tbl_name, 3:list<Partition> new_parts, 4:EnvironmentContext environment_context) throws (1:InvalidOperationException o1, 2:MetaException o2) + + AlterPartitionsResponse alter_partitions_with_environment_context(1:AlterPartitionsRequest req) + throws (1:InvalidOperationException o1, 2:MetaException o2) void alter_partition_with_environment_context(1:string db_name, 2:string tbl_name, 3:Partition new_part, http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 7c7429d..6985736 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -19,11 +19,7 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; -import org.apache.hadoop.hive.metastore.api.ISchemaName; -import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; -import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; +import org.apache.hadoop.hive.metastore.api.*; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -33,58 +29,6 @@ import java.util.Map; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; -import org.apache.hadoop.hive.metastore.api.HiveObjectRef; -import org.apache.hadoop.hive.metastore.api.ISchema; -import org.apache.hadoop.hive.metastore.api.InvalidInputException; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; -import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.NotificationEventRequest; -import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; -import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest; -import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PartitionEventType; -import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse; -import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; -import org.apache.hadoop.hive.metastore.api.PrincipalType; -import org.apache.hadoop.hive.metastore.api.PrivilegeBag; -import org.apache.hadoop.hive.metastore.api.WMNullablePool; -import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; -import org.apache.hadoop.hive.metastore.api.WMResourcePlan; -import org.apache.hadoop.hive.metastore.api.WMTrigger; -import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; -import org.apache.hadoop.hive.metastore.api.Role; -import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; -import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; -import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; -import org.apache.hadoop.hive.metastore.api.SchemaVersion; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TableMeta; -import org.apache.hadoop.hive.metastore.api.Type; -import org.apache.hadoop.hive.metastore.api.UnknownDBException; -import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; -import org.apache.hadoop.hive.metastore.api.UnknownTableException; -import org.apache.hadoop.hive.metastore.api.WMMapping; -import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.ColStatsObjWithSourceInfo; import org.apache.thrift.TException; @@ -247,6 +191,12 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override + public Table getTable(String catName, String dbName, String tableName, long txnId, String writeIdList) + throws MetaException { + return objectStore.getTable(catName, dbName, tableName, txnId, writeIdList); + } + + @Override public boolean addPartition(Partition part) throws InvalidObjectException, MetaException { return objectStore.addPartition(part); @@ -259,6 +209,13 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override + public Partition getPartition(String catName, String dbName, String tableName, + List<String> partVals, long txnId, String writeIdList) + throws MetaException, NoSuchObjectException { + return objectStore.getPartition(catName, dbName, tableName, partVals, txnId, writeIdList); + } + + @Override public boolean dropPartition(String catName, String dbName, String tableName, List<String> partVals) throws MetaException, NoSuchObjectException, InvalidObjectException, InvalidInputException { @@ -343,9 +300,11 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { @Override public void alterPartitions(String catName, String dbName, String tblName, - List<List<String>> partValsList, List<Partition> newParts) + List<List<String>> partValsList, List<Partition> newParts, + long txnId, String writeIdList) throws InvalidObjectException, MetaException { - objectStore.alterPartitions(catName, dbName, tblName, partValsList, newParts); + objectStore.alterPartitions( + catName, dbName, tblName, partValsList, newParts, txnId, writeIdList); } @Override @@ -647,6 +606,15 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override + public ColumnStatistics getTableColumnStatistics(String catName, String dbName, + String tableName, List<String> colNames, + long txnId, String writeIdList) + throws MetaException, NoSuchObjectException { + return objectStore.getTableColumnStatistics( + catName, dbName, tableName, colNames, txnId, writeIdList); + } + + @Override public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, String colName) throws NoSuchObjectException, MetaException, InvalidObjectException, InvalidInputException { @@ -739,6 +707,15 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override + public List<ColumnStatistics> getPartitionColumnStatistics( + String catName, String dbName, String tblName, List<String> partNames, + List<String> colNames, long txnId, String writeIdList) + throws MetaException, NoSuchObjectException { + return objectStore.getPartitionColumnStatistics( + catName, dbName, tblName , colNames, partNames, txnId, writeIdList); + } + + @Override public boolean doesPartitionExist(String catName, String dbName, String tableName, List<FieldSchema> partKeys, List<String> partVals) throws MetaException, NoSuchObjectException { @@ -807,6 +784,15 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override + public AggrStats get_aggr_stats_for(String catName, String dbName, + String tblName, List<String> partNames, + List<String> colNames, + long txnId, String writeIdList) + throws MetaException, NoSuchObjectException { + return null; + } + + @Override public NotificationEventResponse getNextNotification(NotificationEventRequest rqst) { return objectStore.getNextNotification(rqst); } http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index e4f2a17..37e9920 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -19,11 +19,7 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.metastore.api.CreationMetadata; -import org.apache.hadoop.hive.metastore.api.ISchemaName; -import org.apache.hadoop.hive.metastore.api.SchemaVersionDescriptor; -import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; +import org.apache.hadoop.hive.metastore.api.*; import java.nio.ByteBuffer; import java.util.Collections; @@ -31,58 +27,6 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; -import org.apache.hadoop.hive.metastore.api.HiveObjectRef; -import org.apache.hadoop.hive.metastore.api.ISchema; -import org.apache.hadoop.hive.metastore.api.InvalidInputException; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; -import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.NotificationEventRequest; -import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; -import org.apache.hadoop.hive.metastore.api.NotificationEventsCountRequest; -import org.apache.hadoop.hive.metastore.api.NotificationEventsCountResponse; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PartitionEventType; -import org.apache.hadoop.hive.metastore.api.PartitionValuesResponse; -import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; -import org.apache.hadoop.hive.metastore.api.PrincipalType; -import org.apache.hadoop.hive.metastore.api.PrivilegeBag; -import org.apache.hadoop.hive.metastore.api.WMNullablePool; -import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; -import org.apache.hadoop.hive.metastore.api.WMResourcePlan; -import org.apache.hadoop.hive.metastore.api.WMTrigger; -import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; -import org.apache.hadoop.hive.metastore.api.Role; -import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; -import org.apache.hadoop.hive.metastore.api.RuntimeStat; -import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; -import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; -import org.apache.hadoop.hive.metastore.api.SchemaVersion; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TableMeta; -import org.apache.hadoop.hive.metastore.api.Type; -import org.apache.hadoop.hive.metastore.api.UnknownDBException; -import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; -import org.apache.hadoop.hive.metastore.api.UnknownTableException; -import org.apache.hadoop.hive.metastore.api.WMMapping; -import org.apache.hadoop.hive.metastore.api.WMPool; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -243,6 +187,12 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override + public Table getTable(String catalogName, String dbName, String tableName, + long txnid, String writeIdList) throws MetaException { + return null; + } + + @Override public boolean addPartition(Partition part) throws InvalidObjectException, MetaException { return false; @@ -256,6 +206,13 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override + public Partition getPartition(String catName, String dbName, String tableName, List<String> part_vals, + long txnid, String writeIdList) + throws MetaException, NoSuchObjectException { + return null; + } + + @Override public boolean dropPartition(String catName, String dbName, String tableName, List<String> part_vals) throws MetaException { @@ -344,10 +301,10 @@ public class DummyRawStoreForJdoConnection implements RawStore { @Override public void alterPartitions(String catName, String db_name, String tbl_name, - List<List<String>> part_vals_list, List<Partition> new_parts) + List<List<String>> part_vals_list, List<Partition> new_parts, + long txnId, String writeIdList) throws InvalidObjectException, MetaException { - } @Override @@ -700,6 +657,14 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override + public ColumnStatistics getTableColumnStatistics( + String catName, String dbName, String tableName, List<String> colName, + long txnid, String writeIdList) + throws MetaException, NoSuchObjectException { + return null; + } + + @Override public boolean deleteTableColumnStatistics(String catName, String dbName, String tableName, String colName) throws NoSuchObjectException, MetaException, InvalidObjectException { @@ -749,6 +714,14 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override + public List<ColumnStatistics> getPartitionColumnStatistics( + String catName, String dbName, String tblName, List<String> partNames, + List<String> colNames, long txnid, String writeIdList) + throws MetaException, NoSuchObjectException { + return Collections.emptyList(); + } + + @Override public boolean doesPartitionExist(String catName, String dbName, String tableName, List<FieldSchema> partKeys, List<String> partVals) throws MetaException, NoSuchObjectException { @@ -812,6 +785,14 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override + public AggrStats get_aggr_stats_for( + String catName, String dbName, String tblName, List<String> partNames, + List<String> colNames, long txnid, String writeIdList) + throws MetaException, NoSuchObjectException { + return null; + } + + @Override public NotificationEventResponse getNextNotification(NotificationEventRequest rqst) { return null; } http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 2d87a2f..3899f03 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hive.metastore; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; +import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; import java.io.IOException; import java.lang.reflect.Constructor; @@ -1429,6 +1430,17 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos return fastpath ? t : deepCopy(filterHook.filterTable(t)); } + @Override + public Table getTable(String dbName, String tableName, long txnId, String validWriteIdList) + throws MetaException, TException, NoSuchObjectException { + GetTableRequest req = new GetTableRequest(dbName, tableName); + req.setCapabilities(version); + req.setTxnId(txnId); + req.setValidWriteIdList(validWriteIdList); + Table t = client.get_table_req(req).getTable(); + return fastpath ? t : deepCopy(filterHook.filterTable(t)); + } + /** {@inheritDoc} */ @Override public List<Table> getTableObjectsByName(String dbName, List<String> tableNames) @@ -1612,13 +1624,33 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public void alter_partitions(String dbName, String tblName, List<Partition> newParts) throws InvalidOperationException, MetaException, TException { - client.alter_partitions_with_environment_context(dbName, tblName, newParts, null); + client.alter_partitions(dbName, tblName, newParts); } @Override public void alter_partitions(String dbName, String tblName, List<Partition> newParts, EnvironmentContext environmentContext) throws InvalidOperationException, MetaException, TException { - client.alter_partitions_with_environment_context(dbName, tblName, newParts, environmentContext); + AlterPartitionsRequest req = new AlterPartitionsRequest(); + req.setDbName(dbName); + req.setTableName(tblName); + req.setPartitions(newParts); + req.setEnvironmentContext(environmentContext); + client.alter_partitions_with_environment_context(req); + } + + @Override + public void alter_partitions(String dbName, String tblName, List<Partition> newParts, + EnvironmentContext environmentContext, + long txnId, String writeIdList) + throws InvalidOperationException, MetaException, TException { + AlterPartitionsRequest req = new AlterPartitionsRequest(); + req.setDbName(dbName); + req.setTableName(tblName); + req.setPartitions(newParts); + req.setEnvironmentContext(environmentContext); + req.setTxnId(txnId); + req.setValidWriteIdList(writeIdList); + client.alter_partitions_with_environment_context(req); } @Override @@ -1727,6 +1759,17 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos new TableStatsRequest(dbName, tableName, colNames)).getTableStats(); } + @Override + public List<ColumnStatisticsObj> getTableColumnStatistics( + String dbName, String tableName, List<String> colNames, long txnId, String validWriteIdList) + throws NoSuchObjectException, MetaException, TException { + TableStatsRequest tsr = new TableStatsRequest(dbName, tableName, colNames); + tsr.setTxnId(txnId); + tsr.setValidWriteIdList(validWriteIdList); + + return client.get_table_statistics_req(tsr).getTableStats(); + } + /** {@inheritDoc} */ @Override public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics( @@ -1736,6 +1779,18 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos new PartitionsStatsRequest(dbName, tableName, colNames, partNames)).getPartStats(); } + @Override + public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics( + String dbName, String tableName, List<String> partNames, + List<String> colNames, long txnId, String validWriteIdList) + throws NoSuchObjectException, MetaException, TException { + PartitionsStatsRequest psr = new PartitionsStatsRequest(dbName, tableName, colNames, partNames); + psr.setTxnId(txnId); + psr.setValidWriteIdList(validWriteIdList); + return client.get_partitions_statistics_req( + psr).getPartStats(); + } + /** {@inheritDoc} */ @Override public boolean deletePartitionColumnStatistics(String dbName, String tableName, String partName, @@ -2593,6 +2648,21 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } @Override + public AggrStats getAggrColStatsFor( + String dbName, String tblName, List<String> colNames, + List<String> partName, long txnId, String writeIdList) + throws NoSuchObjectException, MetaException, TException { + if (colNames.isEmpty() || partName.isEmpty()) { + LOG.debug("Columns is empty or partNames is empty : Short-circuiting stats eval on client side."); + return new AggrStats(new ArrayList<>(),0); // Nothing to aggregate + } + PartitionsStatsRequest req = new PartitionsStatsRequest(dbName, tblName, colNames, partName); + req.setTxnId(txnId); + req.setValidWriteIdList(writeIdList); + return client.get_aggr_stats_for(req); + } + + @Override public Iterable<Entry<Long, ByteBuffer>> getFileMetadata( final List<Long> fileIds) throws TException { return new MetastoreMapIterable<Long, ByteBuffer>() { @@ -3000,6 +3070,12 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } @Override + public Table getTable(String catName, String dbName, String tableName, + long txnId, String validWriteIdList) throws TException { + throw new UnsupportedOperationException(); + } + + @Override public List<Table> getTableObjectsByName(String catName, String dbName, List<String> tableNames) throws MetaException, InvalidOperationException, UnknownDBException, TException { @@ -3226,7 +3302,8 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public void alter_partitions(String catName, String dbName, String tblName, List<Partition> newParts, - EnvironmentContext environmentContext) throws + EnvironmentContext environmentContext, + long txnId, String writeIdList) throws InvalidOperationException, MetaException, TException { throw new UnsupportedOperationException(); } @@ -3259,6 +3336,14 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } @Override + public List<ColumnStatisticsObj> getTableColumnStatistics( + String catName, String dbName, String tableName, List<String> colNames, + long txnId, String validWriteIdList) + throws NoSuchObjectException, MetaException, TException { + throw new UnsupportedOperationException(); + } + + @Override public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics(String catName, String dbName, String tableName, @@ -3269,6 +3354,14 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } @Override + public Map<String, List<ColumnStatisticsObj>> getPartitionColumnStatistics( + String catName, String dbName, String tableName, List<String> partNames, + List<String> colNames, long txnId, String validWriteIdList) + throws NoSuchObjectException, MetaException, TException { + throw new UnsupportedOperationException(); + } + + @Override public boolean deletePartitionColumnStatistics(String catName, String dbName, String tableName, String partName, String colName) throws NoSuchObjectException, MetaException, InvalidObjectException, TException, @@ -3316,6 +3409,14 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } @Override + public AggrStats getAggrColStatsFor(String catName, String dbName, String tblName, + List<String> colNames, List<String> partNames, + long txnId, String writeIdList) + throws NoSuchObjectException, MetaException, TException { + throw new UnsupportedOperationException(); + } + + @Override public void dropConstraint(String catName, String dbName, String tableName, String constraintName) throws MetaException, NoSuchObjectException, TException { @@ -3420,4 +3521,5 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos public List<RuntimeStat> getRuntimeStats(int maxWeight, int maxCreateTime) throws TException { throw new UnsupportedOperationException(); } + } http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java index 54bf3d7..f19b505 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestAlterPartitions.java @@ -835,7 +835,8 @@ public class TestAlterPartitions extends MetaStoreClientTest { public void testAlterPartitionsWithEnvironmentCtxBogusCatalogName() throws Exception { createTable4PartColsParts(client); Partition part = client.listPartitions(DB_NAME, TABLE_NAME, (short)-1).get(0); - client.alter_partitions("nosuch", DB_NAME, TABLE_NAME, Lists.newArrayList(part), new EnvironmentContext()); + client.alter_partitions("nosuch", DB_NAME, TABLE_NAME, Lists.newArrayList(part), new EnvironmentContext(), + -1, null); } @Test(expected = InvalidOperationException.class) http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java index 9867a81..cfe01fe 100644 --- a/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java +++ b/storage-api/src/java/org/apache/hadoop/hive/common/ValidTxnWriteIdList.java @@ -63,6 +63,10 @@ public class ValidTxnWriteIdList { return null; } + public boolean isEmpty() { + return tablesValidWriteIdList.isEmpty(); + } + // Each ValidWriteIdList is separated with "$" and each one maps to one table // Format <txnId>$<table_name>:<hwm>:<minOpenWriteId>:<open_writeids>:<abort_writeids>$<table_name>... private void readFromString(String src) { http://git-wip-us.apache.org/repos/asf/hive/blob/1d46608e/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java ---------------------------------------------------------------------- diff --git a/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java b/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java index 17f3777..dc50f1b 100644 --- a/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java +++ b/storage-api/src/java/org/apache/hive/common/util/TxnIdUtils.java @@ -36,8 +36,24 @@ public class TxnIdUtils { } return checkEquivalentCommittedIds( - older.getHighWatermark(), older.getInvalidWriteIds(), - newer.getHighWatermark(), newer.getInvalidWriteIds()); + older.getHighWatermark(), older.getInvalidWriteIds(), + newer.getHighWatermark(), newer.getInvalidWriteIds()); + } + + /** + * Check if the give two write id lists are for concurrent writes + * on the table. + */ + public static boolean areTheseConcurrentWrites( + ValidWriteIdList older, ValidWriteIdList newer, long statsWriteId) { + if (!older.getTableName().equalsIgnoreCase(newer.getTableName())) { + return false; + } + + assert(older.getHighWatermark() <= newer.getHighWatermark()); + + // TODO: Just return false for now. + return false; } /**