Repository: hive Updated Branches: refs/heads/master 0fe38792a -> 190c72e77
HIVE-18675 - make HIVE_LOCKS.HL_TXNID NOT NULL (Igor Kryvenko via Eugene Koifman) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/190c72e7 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/190c72e7 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/190c72e7 Branch: refs/heads/master Commit: 190c72e7793d80da48b5c71b5ff0a9547334bedf Parents: 0fe3879 Author: Eugene Koifman <ekoif...@apache.org> Authored: Mon Mar 12 14:05:29 2018 -0700 Committer: Eugene Koifman <ekoif...@apache.org> Committed: Mon Mar 12 14:05:46 2018 -0700 ---------------------------------------------------------------------- metastore/scripts/upgrade/derby/051-HIVE-18675.derby.sql | 1 + .../scripts/upgrade/derby/hive-txn-schema-3.0.0.derby.sql | 6 +++--- .../upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 1 + .../org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java | 8 ++++---- .../src/main/sql/derby/hive-schema-3.0.0.derby.sql | 4 ++-- .../src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql | 2 ++ .../src/main/sql/mssql/hive-schema-3.0.0.mssql.sql | 10 +++++----- .../src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql | 4 +++- .../src/main/sql/mysql/hive-schema-3.0.0.mysql.sql | 4 ++-- .../src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql | 4 +++- .../src/main/sql/oracle/hive-schema-3.0.0.oracle.sql | 2 +- .../src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql | 2 ++ .../src/main/sql/postgres/hive-schema-3.0.0.postgres.sql | 4 ++-- .../main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql | 4 +++- 14 files changed, 34 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/metastore/scripts/upgrade/derby/051-HIVE-18675.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/051-HIVE-18675.derby.sql b/metastore/scripts/upgrade/derby/051-HIVE-18675.derby.sql new file mode 100644 index 0000000..9754292 --- /dev/null +++ b/metastore/scripts/upgrade/derby/051-HIVE-18675.derby.sql @@ -0,0 +1 @@ +ALTER TABLE "APP"."HIVE_LOCKS" ALTER COLUMN "HL_TXNID" NOT NULL; http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/metastore/scripts/upgrade/derby/hive-txn-schema-3.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/hive-txn-schema-3.0.0.derby.sql b/metastore/scripts/upgrade/derby/hive-txn-schema-3.0.0.derby.sql index 2033bdc..6001262 100644 --- a/metastore/scripts/upgrade/derby/hive-txn-schema-3.0.0.derby.sql +++ b/metastore/scripts/upgrade/derby/hive-txn-schema-3.0.0.derby.sql @@ -15,7 +15,7 @@ -- -- Tables for transaction management --- +-- CREATE TABLE TXNS ( TXN_ID bigint PRIMARY KEY, TXN_STATE char(1) NOT NULL, @@ -75,7 +75,7 @@ CREATE UNIQUE INDEX NEXT_WRITE_ID_IDX ON NEXT_WRITE_ID (NWI_DATABASE, NWI_TABLE) CREATE TABLE HIVE_LOCKS ( HL_LOCK_EXT_ID bigint NOT NULL, HL_LOCK_INT_ID bigint NOT NULL, - HL_TXNID bigint, + HL_TXNID bigint NOT NULL, HL_DB varchar(128) NOT NULL, HL_TABLE varchar(128), HL_PARTITION varchar(767), @@ -90,7 +90,7 @@ CREATE TABLE HIVE_LOCKS ( HL_BLOCKEDBY_EXT_ID bigint, HL_BLOCKEDBY_INT_ID bigint, PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) -); +); CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 55b89e7..c2dcb83 100644 --- a/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -8,5 +8,6 @@ RUN '046-HIVE-17566.derby.sql'; RUN '048-HIVE-14498.derby.sql'; RUN '049-HIVE-18489.derby.sql'; RUN '050-HIVE-18192.derby.sql'; +RUN '051-HIVE-18675.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='3.0.0', VERSION_COMMENT='Hive release version 3.0.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java index 88f6346..588f335 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnDbUtil.java @@ -112,7 +112,7 @@ public final class TxnDbUtil { stmt.execute("CREATE TABLE HIVE_LOCKS (" + " HL_LOCK_EXT_ID bigint NOT NULL," + " HL_LOCK_INT_ID bigint NOT NULL," + - " HL_TXNID bigint," + + " HL_TXNID bigint NOT NULL," + " HL_DB varchar(128) NOT NULL," + " HL_TABLE varchar(128)," + " HL_PARTITION varchar(767)," + @@ -149,7 +149,7 @@ public final class TxnDbUtil { stmt.execute("CREATE TABLE NEXT_COMPACTION_QUEUE_ID (NCQ_NEXT bigint NOT NULL)"); stmt.execute("INSERT INTO NEXT_COMPACTION_QUEUE_ID VALUES(1)"); - + stmt.execute("CREATE TABLE COMPLETED_COMPACTIONS (" + " CC_ID bigint PRIMARY KEY," + " CC_DATABASE varchar(128) NOT NULL," + @@ -165,13 +165,13 @@ public final class TxnDbUtil { " CC_HIGHEST_WRITE_ID bigint," + " CC_META_INFO varchar(2048) for bit data," + " CC_HADOOP_JOB_ID varchar(32))"); - + stmt.execute("CREATE TABLE AUX_TABLE (" + " MT_KEY1 varchar(128) NOT NULL," + " MT_KEY2 bigint NOT NULL," + " MT_COMMENT varchar(255)," + " PRIMARY KEY(MT_KEY1, MT_KEY2))"); - + stmt.execute("CREATE TABLE WRITE_SET (" + " WS_DATABASE varchar(128) NOT NULL," + " WS_TABLE varchar(128) NOT NULL," + http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql index 86ef6b1..f6b9fdd 100644 --- a/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/hive-schema-3.0.0.derby.sql @@ -447,7 +447,7 @@ INSERT INTO NEXT_TXN_ID VALUES(1); CREATE TABLE HIVE_LOCKS ( HL_LOCK_EXT_ID bigint NOT NULL, HL_LOCK_INT_ID bigint NOT NULL, - HL_TXNID bigint, + HL_TXNID bigint NOT NULL, HL_DB varchar(128) NOT NULL, HL_TABLE varchar(128), HL_PARTITION varchar(767), @@ -462,7 +462,7 @@ CREATE TABLE HIVE_LOCKS ( HL_BLOCKEDBY_EXT_ID bigint, HL_BLOCKEDBY_INT_ID bigint, PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) -); +); CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS (HL_TXNID); http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql index 1fc34bc..99aed9f 100644 --- a/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql +++ b/standalone-metastore/src/main/sql/derby/upgrade-2.3.0-to-3.0.0.derby.sql @@ -124,3 +124,5 @@ ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint ALTER TABLE "APP"."KEY_CONSTRAINTS" ADD COLUMN "DEFAULT_VALUE" VARCHAR(400); + +ALTER TABLE "APP"."HIVE_LOCKS" ALTER COLUMN "HL_TXNID" NOT NULL; http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql index e0bb25b..0147958 100644 --- a/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/hive-schema-3.0.0.mssql.sql @@ -972,7 +972,7 @@ CREATE TABLE COMPACTION_QUEUE( CQ_HIGHEST_WRITE_ID bigint NULL, CQ_META_INFO varbinary(2048) NULL, CQ_HADOOP_JOB_ID nvarchar(128) NULL, -PRIMARY KEY CLUSTERED +PRIMARY KEY CLUSTERED ( CQ_ID ASC ) @@ -993,7 +993,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_HIGHEST_WRITE_ID bigint NULL, CC_META_INFO varbinary(2048) NULL, CC_HADOOP_JOB_ID nvarchar(128) NULL, -PRIMARY KEY CLUSTERED +PRIMARY KEY CLUSTERED ( CC_ID ASC ) @@ -1013,7 +1013,7 @@ CREATE INDEX COMPLETED_TXN_COMPONENTS_IDX2 ON COMPLETED_TXN_COMPONENTS (CTC_DATA CREATE TABLE HIVE_LOCKS( HL_LOCK_EXT_ID bigint NOT NULL, HL_LOCK_INT_ID bigint NOT NULL, - HL_TXNID bigint NULL, + HL_TXNID bigint NOT NULL, HL_DB nvarchar(128) NOT NULL, HL_TABLE nvarchar(128) NULL, HL_PARTITION nvarchar(767) NULL, @@ -1027,7 +1027,7 @@ CREATE TABLE HIVE_LOCKS( HL_AGENT_INFO nvarchar(128) NULL, HL_BLOCKEDBY_EXT_ID bigint NULL, HL_BLOCKEDBY_INT_ID bigint NULL, -PRIMARY KEY CLUSTERED +PRIMARY KEY CLUSTERED ( HL_LOCK_EXT_ID ASC, HL_LOCK_INT_ID ASC @@ -1062,7 +1062,7 @@ CREATE TABLE TXNS( TXN_AGENT_INFO nvarchar(128) NULL, TXN_META_INFO nvarchar(128) NULL, TXN_HEARTBEAT_COUNT int NULL, -PRIMARY KEY CLUSTERED +PRIMARY KEY CLUSTERED ( TXN_ID ASC ) http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql index 52107b3..3dda4ed 100644 --- a/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql +++ b/standalone-metastore/src/main/sql/mssql/upgrade-2.3.0-to-3.0.0.mssql.sql @@ -174,7 +174,9 @@ EXEC SP_RENAME 'COMPLETED_COMPACTIONS.CC_HIGHEST_TXN_ID', 'CC_HIGHEST_WRITE_ID', -- Modify txn_components/completed_txn_components tables to add write id. ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; - + -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint ALTER TABLE KEY_CONSTRAINTS ADD DEFAULT_VALUE VARCHAR(400); + +ALTER TABLE HIVE_LOCKS MODIFY ALTER COLUMN HL_TXNID bigint NOT NULL; http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql index 566badf..01937ef 100644 --- a/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/hive-schema-3.0.0.mysql.sql @@ -768,7 +768,7 @@ CREATE TABLE IF NOT EXISTS `TYPE_FIELDS` ( ) ENGINE=InnoDB DEFAULT CHARSET=latin1; -- Table `MASTER_KEYS` for classes [org.apache.hadoop.hive.metastore.model.MMasterKey] -CREATE TABLE IF NOT EXISTS `MASTER_KEYS` +CREATE TABLE IF NOT EXISTS `MASTER_KEYS` ( `KEY_ID` INTEGER NOT NULL AUTO_INCREMENT, `MASTER_KEY` VARCHAR(767) BINARY NULL, @@ -988,7 +988,7 @@ INSERT INTO NEXT_TXN_ID VALUES(1); CREATE TABLE HIVE_LOCKS ( HL_LOCK_EXT_ID bigint NOT NULL, HL_LOCK_INT_ID bigint NOT NULL, - HL_TXNID bigint, + HL_TXNID bigint NOT NULL, HL_DB varchar(128) NOT NULL, HL_TABLE varchar(128), HL_PARTITION varchar(767), http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql index 35f08dc..2d4b526 100644 --- a/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql +++ b/standalone-metastore/src/main/sql/mysql/upgrade-2.3.0-to-3.0.0.mysql.sql @@ -159,7 +159,9 @@ ALTER TABLE COMPLETED_COMPACTIONS CHANGE `CC_HIGHEST_TXN_ID` `CC_HIGHEST_WRITE_I -- Modify txn_components/completed_txn_components tables to add write id. ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; - + -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint ALTER TABLE `KEY_CONSTRAINTS` ADD COLUMN `DEFAULT_VALUE` VARCHAR(400); + +ALTER TABLE `HIVE_LOCKS` MODIFY COLUMN `HL_TXNID` NOT NULL; http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql index 82811ee..fdc866d 100644 --- a/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/hive-schema-3.0.0.oracle.sql @@ -962,7 +962,7 @@ INSERT INTO NEXT_TXN_ID VALUES(1); CREATE TABLE HIVE_LOCKS ( HL_LOCK_EXT_ID NUMBER(19) NOT NULL, HL_LOCK_INT_ID NUMBER(19) NOT NULL, - HL_TXNID NUMBER(19), + HL_TXNID NUMBER(19) NOT NULL, HL_DB VARCHAR2(128) NOT NULL, HL_TABLE VARCHAR2(128), HL_PARTITION VARCHAR2(767), http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql index edf14d9..4e3f333 100644 --- a/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql +++ b/standalone-metastore/src/main/sql/oracle/upgrade-2.3.0-to-3.0.0.oracle.sql @@ -187,3 +187,5 @@ ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID number(19); -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint ALTER TABLE KEY_CONSTRAINTS ADD DEFAULT_VALUE VARCHAR(400); + +ALTER TABLE HIVE_LOCKS MODIFY(HL_TXNID NOT NULL); http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql index 543a358..3cc2989 100644 --- a/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/hive-schema-3.0.0.postgres.sql @@ -1654,7 +1654,7 @@ INSERT INTO NEXT_TXN_ID VALUES(1); CREATE TABLE HIVE_LOCKS ( HL_LOCK_EXT_ID bigint NOT NULL, HL_LOCK_INT_ID bigint NOT NULL, - HL_TXNID bigint, + HL_TXNID bigint NOT NULL, HL_DB varchar(128) NOT NULL, HL_TABLE varchar(128), HL_PARTITION varchar(767) DEFAULT NULL, @@ -1669,7 +1669,7 @@ CREATE TABLE HIVE_LOCKS ( HL_BLOCKEDBY_EXT_ID bigint, HL_BLOCKEDBY_INT_ID bigint, PRIMARY KEY(HL_LOCK_EXT_ID, HL_LOCK_INT_ID) -); +); CREATE INDEX HL_TXNID_INDEX ON HIVE_LOCKS USING hash (HL_TXNID); http://git-wip-us.apache.org/repos/asf/hive/blob/190c72e7/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql index ed4ce22..83f71a0 100644 --- a/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql +++ b/standalone-metastore/src/main/sql/postgres/upgrade-2.3.0-to-3.0.0.postgres.sql @@ -198,7 +198,9 @@ ALTER TABLE COMPLETED_COMPACTIONS RENAME CC_HIGHEST_TXN_ID TO CC_HIGHEST_WRITE_I -- Modify txn_components/completed_txn_components tables to add write id. ALTER TABLE TXN_COMPONENTS ADD TC_WRITEID bigint; ALTER TABLE COMPLETED_TXN_COMPONENTS ADD CTC_WRITEID bigint; - + -- HIVE-18726 -- add a new column to support default value for DEFAULT constraint ALTER TABLE "KEY_CONSTRAINTS" ADD COLUMN "DEFAULT_VALUE" VARCHAR(400); + +ALTER TABLE HIVE_LOCKS ALTER COLUMN HL_TXNID SET NOT NULL;