HIVE-14637 : edit or split MoveTask to commit job results to metastore (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/30fd19f4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/30fd19f4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/30fd19f4 Branch: refs/heads/hive-14535 Commit: 30fd19f4b358f12fbbb21d75785817c20e1adf15 Parents: 38409da Author: Sergey Shelukhin <ser...@apache.org> Authored: Wed Sep 7 18:51:39 2016 -0700 Committer: Sergey Shelukhin <ser...@apache.org> Committed: Wed Sep 7 18:51:39 2016 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/history/TestHiveHistory.java | 2 +- metastore/if/hive_metastore.thrift | 36 +- .../upgrade/derby/037-HIVE-14637.derby.sql | 6 + .../upgrade/derby/hive-schema-2.2.0.derby.sql | 12 +- .../derby/upgrade-2.1.0-to-2.2.0.derby.sql | 2 + .../upgrade/mssql/022-HIVE-14637.mssql.sql | 14 + .../upgrade/mssql/hive-schema-2.2.0.mssql.sql | 21 +- .../mssql/upgrade-2.1.0-to-2.2.0.mssql.sql | 2 + .../upgrade/mysql/037-HIVE-14637.mysql.sql | 14 + .../upgrade/mysql/hive-schema-2.2.0.mysql.sql | 15 + .../mysql/upgrade-2.1.0-to-2.2.0.mysql.sql | 2 + .../upgrade/oracle/037-HIVE-14637.oracle.sql | 14 + .../upgrade/oracle/hive-schema-2.2.0.oracle.sql | 18 +- .../oracle/upgrade-2.1.0-to-2.2.0.oracle.sql | 2 + .../postgres/036-HIVE-14637.postgres.sql | 15 + .../postgres/hive-schema-2.2.0.postgres.sql | 22 +- .../upgrade-2.1.0-to-2.2.0.postgres.sql | 2 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 4269 ++++++++----- .../gen/thrift/gen-cpp/ThriftHiveMetastore.h | 378 ++ .../ThriftHiveMetastore_server.skeleton.cpp | 15 + .../thrift/gen-cpp/hive_metastore_constants.cpp | 2 + .../thrift/gen-cpp/hive_metastore_constants.h | 1 + .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 1130 +++- .../gen/thrift/gen-cpp/hive_metastore_types.h | 292 +- .../metastore/api/FinalizeWriteIdRequest.java | 684 ++ .../metastore/api/FinalizeWriteIdResult.java | 283 + .../metastore/api/GetNextWriteIdRequest.java | 490 ++ .../metastore/api/GetNextWriteIdResult.java | 387 ++ .../metastore/api/HeartbeatWriteIdRequest.java | 589 ++ .../metastore/api/HeartbeatWriteIdResult.java | 283 + .../apache/hadoop/hive/metastore/api/Table.java | 206 +- .../hive/metastore/api/ThriftHiveMetastore.java | 5924 +++++++++++++----- .../metastore/api/hive_metastoreConstants.java | 30 +- .../gen-php/metastore/ThriftHiveMetastore.php | 966 ++- .../src/gen/thrift/gen-php/metastore/Types.php | 805 ++- .../hive_metastore/ThriftHiveMetastore-remote | 21 + .../hive_metastore/ThriftHiveMetastore.py | 567 ++ .../thrift/gen-py/hive_metastore/constants.py | 1 + .../gen/thrift/gen-py/hive_metastore/ttypes.py | 478 +- .../thrift/gen-rb/hive_metastore_constants.rb | 2 + .../gen/thrift/gen-rb/hive_metastore_types.rb | 122 +- .../gen/thrift/gen-rb/thrift_hive_metastore.rb | 162 + .../hadoop/hive/metastore/HiveMetaStore.java | 203 +- .../hive/metastore/HiveMetaStoreClient.java | 116 +- .../hadoop/hive/metastore/IMetaStoreClient.java | 7 + .../hadoop/hive/metastore/ObjectStore.java | 116 +- .../apache/hadoop/hive/metastore/RawStore.java | 7 + .../hadoop/hive/metastore/hbase/HBaseStore.java | 19 + .../hadoop/hive/metastore/model/MTable.java | 23 +- .../hive/metastore/model/MTableWrite.java | 67 + metastore/src/model/package.jdo | 29 + .../DummyRawStoreControlledCommit.java | 15 + .../DummyRawStoreForJdoConnection.java | 14 + .../java/org/apache/hadoop/hive/ql/Context.java | 8 +- .../hadoop/hive/ql/exec/FileSinkOperator.java | 22 +- .../apache/hadoop/hive/ql/exec/MoveTask.java | 28 +- .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 10 + .../apache/hadoop/hive/ql/metadata/Hive.java | 118 +- .../hive/ql/optimizer/GenMapRedUtils.java | 1 + .../physical/AnnotateRunTimeStatsOptimizer.java | 3 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 2 + .../hive/ql/parse/ImportSemanticAnalyzer.java | 2 + .../hive/ql/parse/LoadSemanticAnalyzer.java | 1 + .../hadoop/hive/ql/parse/ParseContext.java | 8 - .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 42 +- .../hadoop/hive/ql/parse/TaskCompiler.java | 1 + .../hadoop/hive/ql/plan/FileSinkDesc.java | 24 +- .../hadoop/hive/ql/plan/LoadTableDesc.java | 32 +- .../apache/hadoop/hive/ql/plan/MoveWork.java | 1 - .../hadoop/hive/ql/exec/TestExecDriver.java | 2 +- .../hive/ql/exec/TestFileSinkOperator.java | 2 +- ql/src/test/queries/clientpositive/mm_current.q | 26 +- .../clientpositive/llap/mm_current.q.out | 171 +- 73 files changed, 15320 insertions(+), 4086 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java index 76c1636..0c51a68 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java @@ -103,7 +103,7 @@ public class TestHiveHistory extends TestCase { db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true); db.createTable(src, cols, null, TextInputFormat.class, IgnoreKeyTextOutputFormat.class); - db.loadTable(hadoopDataFile[i], src, false, false, false, false, false); + db.loadTable(hadoopDataFile[i], src, false, false, false, false, false, null); i++; } http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/if/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift index 872c0f3..95eee27 100755 --- a/metastore/if/hive_metastore.thrift +++ b/metastore/if/hive_metastore.thrift @@ -305,7 +305,9 @@ struct Table { 11: string viewExpandedText, // expanded view text, null for non-view 12: string tableType, // table type enum, e.g. EXTERNAL_TABLE 13: optional PrincipalPrivilegeSet privileges, - 14: optional bool temporary=false + 14: optional bool temporary=false, + 15: optional i64 mmNextWriteId, + 16: optional i64 mmWatermarkWriteId } struct Partition { @@ -890,6 +892,33 @@ struct CacheFileMetadataRequest { 4: optional bool isAllParts } + +struct GetNextWriteIdRequest { + 1: required string dbName, + 2: required string tblName +} +struct GetNextWriteIdResult { + 1: required i64 writeId +} + +struct FinalizeWriteIdRequest { + 1: required string dbName, + 2: required string tblName, + 3: required i64 writeId, + 4: required bool commit +} +struct FinalizeWriteIdResult { +} + +struct HeartbeatWriteIdRequest { + 1: required string dbName, + 2: required string tblName, + 3: required i64 writeId +} +struct HeartbeatWriteIdResult { +} + + struct GetAllFunctionsResponse { 1: optional list<Function> functions } @@ -1438,6 +1467,9 @@ service ThriftHiveMetastore extends fb303.FacebookService ClearFileMetadataResult clear_file_metadata(1:ClearFileMetadataRequest req) CacheFileMetadataResult cache_file_metadata(1:CacheFileMetadataRequest req) + GetNextWriteIdResult get_next_write_id(1:GetNextWriteIdRequest req) + FinalizeWriteIdResult finalize_write_id(1:FinalizeWriteIdRequest req) + HeartbeatWriteIdResult heartbeat_write_id(1:HeartbeatWriteIdRequest req) } // * Note about the DDL_TIME: When creating or altering a table or a partition, @@ -1476,5 +1508,7 @@ const string META_TABLE_STORAGE = "storage_handler", const string TABLE_IS_TRANSACTIONAL = "transactional", const string TABLE_NO_AUTO_COMPACT = "no_auto_compaction", const string TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties", +const string TABLE_IS_MM = "hivecommit", + http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql b/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql new file mode 100644 index 0000000..8cea9f1 --- /dev/null +++ b/metastore/scripts/upgrade/derby/037-HIVE-14637.derby.sql @@ -0,0 +1,6 @@ +ALTER TABLE "TBLS" ADD "MM_WATERMARK_WRITE_ID" BIGINT; +ALTER TABLE "TBLS" ADD "MM_NEXT_WRITE_ID" BIGINT; +CREATE TABLE "APP"."TBL_WRITES" ("TW_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, "WRITE_ID" BIGINT NOT NULL, "STATE" CHAR(1) NOT NULL, "LAST_HEARTBEAT" BIGINT); +ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID"); +ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; +CREATE UNIQUE INDEX "APP"."UNIQUEWRITE" ON "APP"."TBL_WRITES" ("TBL_ID", "WRITE_ID"); http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql b/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql index ae980e0..3c4ba4b 100644 --- a/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql +++ b/metastore/scripts/upgrade/derby/hive-schema-2.2.0.derby.sql @@ -60,7 +60,7 @@ CREATE TABLE "APP"."COLUMNS" ("SD_ID" BIGINT NOT NULL, "COMMENT" VARCHAR(256), " CREATE TABLE "APP"."ROLES" ("ROLE_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "OWNER_NAME" VARCHAR(128), "ROLE_NAME" VARCHAR(128)); -CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(128), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR); +CREATE TABLE "APP"."TBLS" ("TBL_ID" BIGINT NOT NULL, "CREATE_TIME" INTEGER NOT NULL, "DB_ID" BIGINT, "LAST_ACCESS_TIME" INTEGER NOT NULL, "OWNER" VARCHAR(767), "RETENTION" INTEGER NOT NULL, "SD_ID" BIGINT, "TBL_NAME" VARCHAR(128), "TBL_TYPE" VARCHAR(128), "VIEW_EXPANDED_TEXT" LONG VARCHAR, "VIEW_ORIGINAL_TEXT" LONG VARCHAR, "MM_WATERMARK_WRITE_ID" BIGINT, "MM_NEXT_WRITE_ID" BIGINT); CREATE TABLE "APP"."PARTITION_KEYS" ("TBL_ID" BIGINT NOT NULL, "PKEY_COMMENT" VARCHAR(4000), "PKEY_NAME" VARCHAR(128) NOT NULL, "PKEY_TYPE" VARCHAR(767) NOT NULL, "INTEGER_IDX" INTEGER NOT NULL); @@ -112,6 +112,16 @@ ALTER TABLE "APP"."KEY_CONSTRAINTS" ADD CONSTRAINT "CONSTRAINTS_PK" PRIMARY KEY CREATE INDEX "APP"."CONSTRAINTS_PARENT_TBL_ID_INDEX" ON "APP"."KEY_CONSTRAINTS"("PARENT_TBL_ID"); +CREATE TABLE "APP"."TBL_WRITES" ("TW_ID" BIGINT NOT NULL, "TBL_ID" BIGINT NOT NULL, "WRITE_ID" BIGINT NOT NULL, "STATE" CHAR(1) NOT NULL, "LAST_HEARTBEAT" BIGINT); + +ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID"); + +ALTER TABLE "APP"."TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "APP"."TBLS" ("TBL_ID") ON DELETE NO ACTION ON UPDATE NO ACTION; + +CREATE UNIQUE INDEX "APP"."UNIQUEWRITE" ON "APP"."TBL_WRITES" ("TBL_ID", "WRITE_ID"); + + + -- ---------------------------------------------- -- DDL Statements for indexes -- ---------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql index 25a5e37..67750a6 100644 --- a/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-2.1.0-to-2.2.0.derby.sql @@ -1,3 +1,5 @@ -- Upgrade MetaStore schema from 2.1.0 to 2.2.0 +RUN '037-HIVE-14637.derby.sql'; + UPDATE "APP".VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql b/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql new file mode 100644 index 0000000..bb42935 --- /dev/null +++ b/metastore/scripts/upgrade/mssql/022-HIVE-14637.mssql.sql @@ -0,0 +1,14 @@ +ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID BIGINT NULL; +ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID BIGINT NULL; + +CREATE TABLE TBL_WRITES +( + TW_ID BIGINT NOT NULL, + TBL_ID BIGINT NOT NULL, + WRITE_ID BIGINT NOT NULL, + STATE CHAR(1) NOT NULL, + LAST_HEARTBEAT BIGINT +); +ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID); +ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ; +CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID); http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql index 8735b50..6bd0d87 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.2.0.mssql.sql @@ -358,7 +358,9 @@ CREATE TABLE TBLS TBL_NAME nvarchar(128) NULL, TBL_TYPE nvarchar(128) NULL, VIEW_EXPANDED_TEXT text NULL, - VIEW_ORIGINAL_TEXT text NULL + VIEW_ORIGINAL_TEXT text NULL, + MM_WATERMARK_WRITE_ID BIGINT NULL, + MM_NEXT_WRITE_ID BIGINT NULL ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); @@ -591,6 +593,23 @@ CREATE TABLE NOTIFICATION_SEQUENCE ALTER TABLE NOTIFICATION_SEQUENCE ADD CONSTRAINT NOTIFICATION_SEQUENCE_PK PRIMARY KEY (NNI_ID); + +CREATE TABLE TBL_WRITES +( + TW_ID BIGINT NOT NULL, + TBL_ID BIGINT NOT NULL, + WRITE_ID BIGINT NOT NULL, + STATE CHAR(1) NOT NULL, + LAST_HEARTBEAT BIGINT +); + +ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID); + +ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) ; + +CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID); + + -- Constraints for table MASTER_KEYS for class(es) [org.apache.hadoop.hive.metastore.model.MMasterKey] -- Constraints for table IDXS for class(es) [org.apache.hadoop.hive.metastore.model.MIndex] http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql index df97206..2e6f394 100644 --- a/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/upgrade-2.1.0-to-2.2.0.mssql.sql @@ -1,4 +1,6 @@ SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS MESSAGE; +:r 022-HIVE-14637.mssql.sql + UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql b/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql new file mode 100644 index 0000000..1b740d5 --- /dev/null +++ b/metastore/scripts/upgrade/mysql/037-HIVE-14637.mysql.sql @@ -0,0 +1,14 @@ +alter table `TBLS` ADD COLUMN `MM_WATERMARK_WRITE_ID` bigint(20); +alter table `TBLS` ADD COLUMN `MM_NEXT_WRITE_ID` bigint(20); + +CREATE TABLE IF NOT EXISTS `TBL_WRITES` +( + `TW_ID` BIGINT NOT NULL, + `TBL_ID` BIGINT NOT NULL, + `WRITE_ID` BIGINT NOT NULL, + `STATE` CHAR(1) NOT NULL, + `LAST_HEARTBEAT` BIGINT, + PRIMARY KEY (`TW_ID`), + UNIQUE KEY `UNIQUEWRITE` (`TBL_ID`,`WRITE_ID`), + CONSTRAINT `TBL_WRITES_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql index 91e221d..f7ef948 100644 --- a/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/hive-schema-2.2.0.mysql.sql @@ -587,6 +587,8 @@ CREATE TABLE IF NOT EXISTS `TBLS` ( `TBL_TYPE` varchar(128) CHARACTER SET latin1 COLLATE latin1_bin DEFAULT NULL, `VIEW_EXPANDED_TEXT` mediumtext, `VIEW_ORIGINAL_TEXT` mediumtext, + `MM_WATERMARK_WRITE_ID` bigint(20), + `MM_NEXT_WRITE_ID` bigint(20), PRIMARY KEY (`TBL_ID`), UNIQUE KEY `UNIQUETABLE` (`TBL_NAME`,`DB_ID`), KEY `TBLS_N50` (`SD_ID`), @@ -827,6 +829,19 @@ CREATE TABLE IF NOT EXISTS `KEY_CONSTRAINTS` CREATE INDEX `CONSTRAINTS_PARENT_TABLE_ID_INDEX` ON KEY_CONSTRAINTS (`PARENT_TBL_ID`) USING BTREE; +CREATE TABLE IF NOT EXISTS `TBL_WRITES` +( + `TW_ID` BIGINT NOT NULL, + `TBL_ID` BIGINT NOT NULL, + `WRITE_ID` BIGINT NOT NULL, + `STATE` CHAR(1) NOT NULL, + `LAST_HEARTBEAT` BIGINT, + PRIMARY KEY (`TW_ID`), + UNIQUE KEY `UNIQUEWRITE` (`TBL_ID`,`WRITE_ID`), + CONSTRAINT `TBL_WRITES_FK1` FOREIGN KEY (`TBL_ID`) REFERENCES `TBLS` (`TBL_ID`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + + -- ---------------------------- -- Transaction and Lock Tables -- ---------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql index de38b58..6ac1b89 100644 --- a/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/upgrade-2.1.0-to-2.2.0.mysql.sql @@ -1,5 +1,7 @@ SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS ' '; +SOURCE 037-HIVE-14637.mysql.sql; + UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS ' '; http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql b/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql new file mode 100644 index 0000000..bc5fb6b --- /dev/null +++ b/metastore/scripts/upgrade/oracle/037-HIVE-14637.oracle.sql @@ -0,0 +1,14 @@ +ALTER TABLE TBLS ADD MM_WATERMARK_WRITE_ID NUMBER; +ALTER TABLE TBLS ADD MM_NEXT_WRITE_ID NUMBER; + +CREATE TABLE TBL_WRITES +( + TW_ID NUMBER NOT NULL, + TBL_ID NUMBER NOT NULL, + WRITE_ID NUMBER NOT NULL, + STATE CHAR(1) NOT NULL, + LAST_HEARTBEAT NUMBER NOT NULL +); +ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID); +ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; +CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID); http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql index 39ba7cb..503ce09 100644 --- a/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/hive-schema-2.2.0.oracle.sql @@ -375,7 +375,9 @@ CREATE TABLE TBLS TBL_NAME VARCHAR2(128) NULL, TBL_TYPE VARCHAR2(128) NULL, VIEW_EXPANDED_TEXT CLOB NULL, - VIEW_ORIGINAL_TEXT CLOB NULL + VIEW_ORIGINAL_TEXT CLOB NULL, + MM_WATERMARK_WRITE_ID NUMBER NULL, + MM_NEXT_WRITE_ID NUMBER NULL ); ALTER TABLE TBLS ADD CONSTRAINT TBLS_PK PRIMARY KEY (TBL_ID); @@ -797,6 +799,20 @@ ALTER TABLE KEY_CONSTRAINTS ADD CONSTRAINT CONSTRAINTS_PK PRIMARY KEY (CONSTRAIN CREATE INDEX CONSTRAINTS_PT_INDEX ON KEY_CONSTRAINTS(PARENT_TBL_ID); +CREATE TABLE TBL_WRITES +( + TW_ID NUMBER NOT NULL, + TBL_ID NUMBER NOT NULL, + WRITE_ID NUMBER NOT NULL, + STATE CHAR(1) NOT NULL, + LAST_HEARTBEAT NUMBER NOT NULL +); + +ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_PK PRIMARY KEY (TW_ID); + +ALTER TABLE TBL_WRITES ADD CONSTRAINT TBL_WRITES_FK1 FOREIGN KEY (TBL_ID) REFERENCES TBLS (TBL_ID) INITIALLY DEFERRED ; + +CREATE UNIQUE INDEX UNIQUEWRITE ON TBL_WRITES (TBL_ID, WRITE_ID); ------------------------------ -- Transaction and lock tables http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql index 66784a4..8d841d6 100644 --- a/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/upgrade-2.1.0-to-2.2.0.oracle.sql @@ -1,4 +1,6 @@ SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0' AS Status from dual; +@037-HIVE-14637.oracle.sql; + UPDATE VERSION SET SCHEMA_VERSION='2.2.0', VERSION_COMMENT='Hive release version 2.2.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0' AS Status from dual; http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql b/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql new file mode 100644 index 0000000..d94c19d --- /dev/null +++ b/metastore/scripts/upgrade/postgres/036-HIVE-14637.postgres.sql @@ -0,0 +1,15 @@ + +ALTER TABLE "TBLS" ADD COLUMN "MM_WATERMARK_WRITE_ID" bigint; +ALTER TABLE "TBLS" ADD COLUMN "MM_NEXT_WRITE_ID" bigint; + +CREATE TABLE "TBL_WRITES" +( + "TW_ID" BIGINT NOT NULL, + "TBL_ID" BIGINT NOT NULL, + "WRITE_ID" BIGINT NOT NULL, + "STATE" CHAR(1) NOT NULL, + "LAST_HEARTBEAT" BIGINT NOT NULL +); +ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID"); +ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS" ("TBL_ID") DEFERRABLE; +ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "UNIQUEWRITE" UNIQUE ("TBL_ID", "WRITE_ID"); http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql index 63ac3be..bf1d769 100644 --- a/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/hive-schema-2.2.0.postgres.sql @@ -372,7 +372,9 @@ CREATE TABLE "TBLS" ( "TBL_NAME" character varying(128) DEFAULT NULL::character varying, "TBL_TYPE" character varying(128) DEFAULT NULL::character varying, "VIEW_EXPANDED_TEXT" text, - "VIEW_ORIGINAL_TEXT" text + "VIEW_ORIGINAL_TEXT" text, + "MM_WATERMARK_WRITE_ID" bigint, + "MM_NEXT_WRITE_ID" bigint ); @@ -604,6 +606,24 @@ CREATE TABLE "KEY_CONSTRAINTS" CREATE INDEX "CONSTRAINTS_PARENT_TBLID_INDEX" ON "KEY_CONSTRAINTS" USING BTREE ("PARENT_TBL_ID"); + + +CREATE TABLE "TBL_WRITES" +( + "TW_ID" BIGINT NOT NULL, + "TBL_ID" BIGINT NOT NULL, + "WRITE_ID" BIGINT NOT NULL, + "STATE" CHAR(1) NOT NULL, + "LAST_HEARTBEAT" BIGINT NOT NULL +); + +ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_PK" PRIMARY KEY ("TW_ID"); + +ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "TBL_WRITES_FK1" FOREIGN KEY ("TBL_ID") REFERENCES "TBLS" ("TBL_ID") DEFERRABLE; + +ALTER TABLE ONLY "TBL_WRITES" ADD CONSTRAINT "UNIQUEWRITE" UNIQUE ("TBL_ID", "WRITE_ID"); + + -- -- Name: BUCKETING_COLS_pkey; Type: CONSTRAINT; Schema: public; Owner: hiveuser; Tablespace: -- http://git-wip-us.apache.org/repos/asf/hive/blob/30fd19f4/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql index 0b4591d..70542b8 100644 --- a/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/upgrade-2.1.0-to-2.2.0.postgres.sql @@ -1,5 +1,7 @@ SELECT 'Upgrading MetaStore schema from 2.1.0 to 2.2.0'; +\i 036-HIVE-14637.postgres.sql; + UPDATE "VERSION" SET "SCHEMA_VERSION"='2.2.0', "VERSION_COMMENT"='Hive release version 2.2.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.1.0 to 2.2.0';
