HIVE-13354 : Add ability to specify Compaction options per table and per request (Wei Zheng, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c57a5961 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c57a5961 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c57a5961 Branch: refs/heads/master Commit: c57a59611fa168ee38c6ee0ee60b1d6c4994f9f8 Parents: 793681c Author: Wei Zheng <w...@apache.org> Authored: Fri May 27 11:20:14 2016 -0700 Committer: Wei Zheng <w...@apache.org> Committed: Fri May 27 11:20:14 2016 -0700 ---------------------------------------------------------------------- .../hive/ql/txn/compactor/TestCompactor.java | 161 ++ metastore/if/hive_metastore.thrift | 1 + .../upgrade/derby/036-HIVE-13354.derby.sql | 2 + .../derby/hive-txn-schema-1.3.0.derby.sql | 2 + .../derby/hive-txn-schema-2.1.0.derby.sql | 2 + .../derby/upgrade-1.2.0-to-1.3.0.derby.sql | 1 + .../derby/upgrade-2.0.0-to-2.1.0.derby.sql | 1 + .../upgrade/mssql/021-HIVE-13354.mssql.sql | 2 + .../upgrade/mssql/hive-schema-1.3.0.mssql.sql | 2 + .../upgrade/mssql/hive-schema-2.1.0.mssql.sql | 2 + .../mssql/upgrade-1.2.0-to-1.3.0.mssql.sql | 1 + .../mssql/upgrade-2.0.0-to-2.1.0.mssql.sql | 1 + .../upgrade/mysql/036-HIVE-13354.mysql.sql | 2 + .../mysql/hive-txn-schema-1.3.0.mysql.sql | 2 + .../mysql/hive-txn-schema-2.1.0.mysql.sql | 2 + .../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 1 + .../mysql/upgrade-2.0.0-to-2.1.0.mysql.sql | 1 + .../upgrade/oracle/036-HIVE-13354.oracle.sql | 2 + .../oracle/hive-txn-schema-1.3.0.oracle.sql | 2 + .../oracle/hive-txn-schema-2.1.0.oracle.sql | 2 + .../oracle/upgrade-1.2.0-to-1.3.0.oracle.sql | 1 + .../oracle/upgrade-2.0.0-to-2.1.0.oracle.sql | 1 + .../postgres/035-HIVE-13354.postgres.sql | 2 + .../postgres/hive-txn-schema-1.3.0.postgres.sql | 2 + .../postgres/hive-txn-schema-2.1.0.postgres.sql | 2 + .../upgrade-1.2.0-to-1.3.0.postgres.sql | 1 + .../upgrade-2.0.0-to-2.1.0.postgres.sql | 1 + .../gen/thrift/gen-cpp/ThriftHiveMetastore.cpp | 2020 ++++++++-------- .../gen/thrift/gen-cpp/hive_metastore_types.cpp | 980 ++++---- .../gen/thrift/gen-cpp/hive_metastore_types.h | 10 +- .../metastore/api/AddDynamicPartitions.java | 32 +- .../metastore/api/ClearFileMetadataRequest.java | 32 +- .../hive/metastore/api/CompactionRequest.java | 166 +- .../hive/metastore/api/FireEventRequest.java | 32 +- .../metastore/api/GetAllFunctionsResponse.java | 36 +- .../api/GetFileMetadataByExprRequest.java | 32 +- .../api/GetFileMetadataByExprResult.java | 48 +- .../metastore/api/GetFileMetadataRequest.java | 32 +- .../metastore/api/GetFileMetadataResult.java | 44 +- .../metastore/api/InsertEventRequestData.java | 32 +- .../api/NotificationEventResponse.java | 36 +- .../metastore/api/PutFileMetadataRequest.java | 64 +- .../hive/metastore/api/ShowCompactResponse.java | 36 +- .../hive/metastore/api/ThriftHiveMetastore.java | 2188 +++++++++--------- .../gen-php/metastore/ThriftHiveMetastore.php | 1232 +++++----- .../src/gen/thrift/gen-php/metastore/Types.php | 315 +-- .../hive_metastore/ThriftHiveMetastore.py | 830 +++---- .../gen/thrift/gen-py/hive_metastore/ttypes.py | 197 +- .../gen/thrift/gen-rb/hive_metastore_types.rb | 4 +- .../hive/metastore/HiveMetaStoreClient.java | 14 + .../hadoop/hive/metastore/IMetaStoreClient.java | 18 + .../hive/metastore/txn/CompactionInfo.java | 30 +- .../metastore/txn/CompactionTxnHandler.java | 11 +- .../hadoop/hive/metastore/txn/TxnDbUtil.java | 2 + .../hadoop/hive/metastore/txn/TxnHandler.java | 8 + .../hadoop/hive/metastore/txn/TxnUtils.java | 56 +- .../org/apache/hadoop/hive/ql/exec/DDLTask.java | 2 +- .../apache/hadoop/hive/ql/metadata/Hive.java | 6 +- .../hive/ql/parse/DDLSemanticAnalyzer.java | 5 + .../apache/hadoop/hive/ql/parse/HiveParser.g | 4 +- .../hive/ql/plan/AlterTableSimpleDesc.java | 8 + .../hive/ql/txn/compactor/CompactorMR.java | 121 +- .../hadoop/hive/ql/txn/compactor/Initiator.java | 22 +- .../hadoop/hive/ql/txn/compactor/Worker.java | 9 + .../hive/ql/txn/compactor/TestWorker.java | 9 +- 65 files changed, 4780 insertions(+), 4145 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java index ad32074..e6ccdbc 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCompactor.java @@ -31,6 +31,7 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.CompactionRequest; import org.apache.hadoop.hive.metastore.api.CompactionType; import org.apache.hadoop.hive.metastore.api.LongColumnStatsData; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.ShowCompactRequest; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; @@ -51,6 +52,7 @@ import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat; import org.apache.hadoop.hive.ql.io.orc.OrcStruct; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.hadoop.mapred.JobConf; import org.apache.hive.hcatalog.common.HCatUtil; import org.apache.hive.hcatalog.streaming.DelimitedInputWriter; import org.apache.hive.hcatalog.streaming.HiveEndPoint; @@ -78,6 +80,7 @@ import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.TimeUnit; /** */ @@ -852,6 +855,134 @@ public class TestCompactor { connection.close(); } } + + /** + * Users have the choice of specifying compaction related tblproperties either in CREATE TABLE + * statement or in ALTER TABLE .. COMPACT statement. This tests both cases. + * @throws Exception + */ + @Test + public void testTableProperties() throws Exception { + String tblName1 = "ttp1"; // plain acid table + String tblName2 = "ttp2"; // acid table with customized tblproperties + executeStatementOnDriver("drop table if exists " + tblName1, driver); + executeStatementOnDriver("drop table if exists " + tblName2, driver); + executeStatementOnDriver("CREATE TABLE " + tblName1 + "(a INT, b STRING) " + + " CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES ('transactional'='true')", driver); + executeStatementOnDriver("CREATE TABLE " + tblName2 + "(a INT, b STRING) " + + " CLUSTERED BY(a) INTO 2 BUCKETS STORED AS ORC TBLPROPERTIES (" + + "'transactional'='true'," + + "'compactor.mapreduce.map.memory.mb'='2048'," + // 2048 MB memory for compaction map job + "'compactorthreshold.hive.compactor.delta.num.threshold'='4'," + // minor compaction if more than 4 delta dirs + "'compactorthreshold.hive.compactor.delta.pct.threshold'='0.5'" + // major compaction if more than 50% + ")", driver); + + // Insert 5 rows to both tables + executeStatementOnDriver("insert into " + tblName1 + " values (1, 'a')", driver); + executeStatementOnDriver("insert into " + tblName1 + " values (2, 'b')", driver); + executeStatementOnDriver("insert into " + tblName1 + " values (3, 'c')", driver); + executeStatementOnDriver("insert into " + tblName1 + " values (4, 'd')", driver); + executeStatementOnDriver("insert into " + tblName1 + " values (5, 'e')", driver); + + executeStatementOnDriver("insert into " + tblName2 + " values (1, 'a')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (2, 'b')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (3, 'c')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (4, 'd')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (5, 'e')", driver); + + runInitiator(conf); + + // Compactor should only schedule compaction for ttp2 (delta.num.threshold=4), not ttp1 + TxnStore txnHandler = TxnUtils.getTxnStore(conf); + ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(1, rsp.getCompacts().size()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(0).getState()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(CompactionType.MAJOR, rsp.getCompacts().get(0).getType()); // type is MAJOR since there's no base yet + + // Finish the scheduled compaction for ttp2, and manually compact ttp1, to make them comparable again + executeStatementOnDriver("alter table " + tblName1 + " compact 'major'", driver); + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(2, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(0).getState()); + Assert.assertEquals("ttp1", rsp.getCompacts().get(1).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(1).getState()); + // compact ttp2, by running the Worker explicitly, in order to get the reference to the compactor MR job + AtomicBoolean stop = new AtomicBoolean(true); + Worker t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(conf); + AtomicBoolean looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + JobConf job = t.getMrJob(); + Assert.assertEquals("2048", job.get("mapreduce.map.memory.mb")); // 2048 comes from tblproperties + // Compact ttp1 + stop = new AtomicBoolean(true); + t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(conf); + looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + job = t.getMrJob(); + Assert.assertEquals("1024", job.get("mapreduce.map.memory.mb")); // 1024 is the default value + // Clean up + runCleaner(conf); + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(2, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, rsp.getCompacts().get(0).getState()); + Assert.assertEquals("ttp1", rsp.getCompacts().get(1).getTablename()); + Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, rsp.getCompacts().get(1).getState()); + + // Insert one more row - this should trigger hive.compactor.delta.pct.threshold to be reached for ttp2 + executeStatementOnDriver("insert into " + tblName1 + " values (6, 'f')", driver); + executeStatementOnDriver("insert into " + tblName2 + " values (6, 'f')", driver); + + // Intentionally set this high so that it will not trigger major compaction for ttp1. + // Only trigger major compaction for ttp2 (delta.pct.threshold=0.5) because of the newly inserted row (actual pct: 0.66) + conf.setFloatVar(HiveConf.ConfVars.HIVE_COMPACTOR_DELTA_PCT_THRESHOLD, 0.8f); + runInitiator(conf); + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(3, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(0).getState()); + + // Finish the scheduled compaction for ttp2 + runWorker(conf); + runCleaner(conf); + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(3, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.SUCCEEDED_RESPONSE, rsp.getCompacts().get(0).getState()); + + // Now test tblproperties specified on ALTER TABLE .. COMPACT .. statement + executeStatementOnDriver("insert into " + tblName2 + " values (7, 'g')", driver); + executeStatementOnDriver("alter table " + tblName2 + " compact 'major'" + + " with overwrite tblproperties (" + + "'compactor.mapreduce.map.memory.mb'='3072'," + + "'tblprops.orc.compress.size'='8192')", driver); + + rsp = txnHandler.showCompact(new ShowCompactRequest()); + Assert.assertEquals(4, rsp.getCompacts().size()); + Assert.assertEquals("ttp2", rsp.getCompacts().get(0).getTablename()); + Assert.assertEquals(TxnStore.INITIATED_RESPONSE, rsp.getCompacts().get(0).getState()); + + // Run the Worker explicitly, in order to get the reference to the compactor MR job + stop = new AtomicBoolean(true); + t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(conf); + looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + job = t.getMrJob(); + Assert.assertEquals("3072", job.get("mapreduce.map.memory.mb")); + Assert.assertTrue(job.get("hive.compactor.table.props").contains("orc.compress.size4:8192")); + } + private void writeBatch(StreamingConnection connection, DelimitedInputWriter writer, boolean closeEarly) throws InterruptedException, StreamingException { @@ -975,4 +1106,34 @@ public class TestCompactor { } } + + static void runInitiator(HiveConf hiveConf) throws MetaException { + AtomicBoolean stop = new AtomicBoolean(true); + Initiator t = new Initiator(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + } + + static void runWorker(HiveConf hiveConf) throws MetaException { + AtomicBoolean stop = new AtomicBoolean(true); + Worker t = new Worker(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + } + + static void runCleaner(HiveConf hiveConf) throws MetaException { + AtomicBoolean stop = new AtomicBoolean(true); + Cleaner t = new Cleaner(); + t.setThreadId((int) t.getId()); + t.setHiveConf(hiveConf); + AtomicBoolean looped = new AtomicBoolean(); + t.init(stop, looped); + t.run(); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/if/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/metastore/if/hive_metastore.thrift b/metastore/if/hive_metastore.thrift index 738456c..4d92b73 100755 --- a/metastore/if/hive_metastore.thrift +++ b/metastore/if/hive_metastore.thrift @@ -744,6 +744,7 @@ struct CompactionRequest { 3: optional string partitionname, 4: required CompactionType type, 5: optional string runas, + 6: optional map<string, string> properties } struct ShowCompactRequest { http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/derby/036-HIVE-13354.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/036-HIVE-13354.derby.sql b/metastore/scripts/upgrade/derby/036-HIVE-13354.derby.sql new file mode 100644 index 0000000..2f691b1 --- /dev/null +++ b/metastore/scripts/upgrade/derby/036-HIVE-13354.derby.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES varchar(2048); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048); http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql b/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql index 480c19e..634dd73 100644 --- a/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql +++ b/metastore/scripts/upgrade/derby/hive-txn-schema-1.3.0.derby.sql @@ -82,6 +82,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -102,6 +103,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/derby/hive-txn-schema-2.1.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/hive-txn-schema-2.1.0.derby.sql b/metastore/scripts/upgrade/derby/hive-txn-schema-2.1.0.derby.sql index 11d86ca..b31ea6e 100644 --- a/metastore/scripts/upgrade/derby/hive-txn-schema-2.1.0.derby.sql +++ b/metastore/scripts/upgrade/derby/hive-txn-schema-2.1.0.derby.sql @@ -82,6 +82,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -102,6 +103,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql index 6b90b73..076afdd 100644 --- a/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-1.2.0-to-1.3.0.derby.sql @@ -11,5 +11,6 @@ RUN '030-HIVE-12823.derby.sql'; RUN '031-HIVE-12831.derby.sql'; RUN '032-HIVE-12832.derby.sql'; RUN '035-HIVE-13395.derby.sql'; +RUN '036-HIVE-13354.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/derby/upgrade-2.0.0-to-2.1.0.derby.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/derby/upgrade-2.0.0-to-2.1.0.derby.sql b/metastore/scripts/upgrade/derby/upgrade-2.0.0-to-2.1.0.derby.sql index 9c730af..08be7fd 100644 --- a/metastore/scripts/upgrade/derby/upgrade-2.0.0-to-2.1.0.derby.sql +++ b/metastore/scripts/upgrade/derby/upgrade-2.0.0-to-2.1.0.derby.sql @@ -1,5 +1,6 @@ -- Upgrade MetaStore schema from 2.0.0 to 2.1.0 RUN '034-HIVE-13076.derby.sql'; RUN '035-HIVE-13395.derby.sql'; +RUN '036-HIVE-13354.derby.sql'; UPDATE "APP".VERSION SET SCHEMA_VERSION='2.1.0', VERSION_COMMENT='Hive release version 2.1.0' where VER_ID=1; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mssql/021-HIVE-13354.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/021-HIVE-13354.mssql.sql b/metastore/scripts/upgrade/mssql/021-HIVE-13354.mssql.sql new file mode 100644 index 0000000..518b142 --- /dev/null +++ b/metastore/scripts/upgrade/mssql/021-HIVE-13354.mssql.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES nvarchar(2048) NULL; +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES nvarchar(2048) NULL; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql index a184f24..022a474 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-1.3.0.mssql.sql @@ -863,6 +863,7 @@ CREATE TABLE COMPACTION_QUEUE( CQ_PARTITION nvarchar(767) NULL, CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES nvarchar(2048) NULL, CQ_WORKER_ID nvarchar(128) NULL, CQ_START bigint NULL, CQ_RUN_AS nvarchar(128) NULL, @@ -882,6 +883,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION nvarchar(767) NULL, CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES nvarchar(2048) NULL, CC_WORKER_ID nvarchar(128) NULL, CC_START bigint NULL, CC_END bigint NULL, http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql b/metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql index 5d90cfc..5918480 100644 --- a/metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/hive-schema-2.1.0.mssql.sql @@ -863,6 +863,7 @@ CREATE TABLE COMPACTION_QUEUE( CQ_PARTITION nvarchar(767) NULL, CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES nvarchar(2048) NULL, CQ_WORKER_ID nvarchar(128) NULL, CQ_START bigint NULL, CQ_RUN_AS nvarchar(128) NULL, @@ -882,6 +883,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION nvarchar(767) NULL, CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES nvarchar(2048) NULL, CC_WORKER_ID nvarchar(128) NULL, CC_START bigint NULL, CC_END bigint NULL, http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql index 208ddfe..e62c0fd 100644 --- a/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/upgrade-1.2.0-to-1.3.0.mssql.sql @@ -12,6 +12,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE; :r 016-HIVE-12831.mssql.sql :r 017-HIVE-12832.mssql.sql :r 020-HIVE-13395.mssql.sql +:r 021-HIVE-13354.mssql.sql UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mssql/upgrade-2.0.0-to-2.1.0.mssql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mssql/upgrade-2.0.0-to-2.1.0.mssql.sql b/metastore/scripts/upgrade/mssql/upgrade-2.0.0-to-2.1.0.mssql.sql index 833e1a5..dc30d24 100644 --- a/metastore/scripts/upgrade/mssql/upgrade-2.0.0-to-2.1.0.mssql.sql +++ b/metastore/scripts/upgrade/mssql/upgrade-2.0.0-to-2.1.0.mssql.sql @@ -2,6 +2,7 @@ SELECT 'Upgrading MetaStore schema from 2.0.0 to 2.1.0' AS MESSAGE; :r 019-HIVE-13076.mssql.sql :r 020-HIVE-13395.mssql.sql +:r 021-HIVE-13354.mssql.sql UPDATE VERSION SET SCHEMA_VERSION='2.1.0', VERSION_COMMENT='Hive release version 2.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.0.0 to 2.1.0' AS MESSAGE; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mysql/036-HIVE-13354.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/036-HIVE-13354.mysql.sql b/metastore/scripts/upgrade/mysql/036-HIVE-13354.mysql.sql new file mode 100644 index 0000000..2f691b1 --- /dev/null +++ b/metastore/scripts/upgrade/mysql/036-HIVE-13354.mysql.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES varchar(2048); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048); http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql index d873012..f234666 100644 --- a/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/hive-txn-schema-1.3.0.mysql.sql @@ -85,6 +85,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -100,6 +101,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mysql/hive-txn-schema-2.1.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/hive-txn-schema-2.1.0.mysql.sql b/metastore/scripts/upgrade/mysql/hive-txn-schema-2.1.0.mysql.sql index 369d6bb..58835cb 100644 --- a/metastore/scripts/upgrade/mysql/hive-txn-schema-2.1.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/hive-txn-schema-2.1.0.mysql.sql @@ -85,6 +85,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -100,6 +101,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql index b65aee5..fe5b79d 100644 --- a/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/upgrade-1.2.0-to-1.3.0.mysql.sql @@ -12,6 +12,7 @@ SOURCE 030-HIVE-12823.mysql.sql; SOURCE 031-HIVE-12831.mysql.sql; SOURCE 032-HIVE-12832.mysql.sql; SOURCE 035-HIVE-13395.mysql.sql; +SOURCE 036-HIVE-13354.mysql.sql; UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS ' '; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql b/metastore/scripts/upgrade/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql index 57191cc..226f4b4 100644 --- a/metastore/scripts/upgrade/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql +++ b/metastore/scripts/upgrade/mysql/upgrade-2.0.0-to-2.1.0.mysql.sql @@ -2,6 +2,7 @@ SELECT 'Upgrading MetaStore schema from 2.0.0 to 2.1.0' AS ' '; SOURCE 034-HIVE-13076.mysql.sql; SOURCE 035-HIVE-13395.mysql.sql; +SOURCE 036-HIVE-13354.mysql.sql; UPDATE VERSION SET SCHEMA_VERSION='2.1.0', VERSION_COMMENT='Hive release version 2.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.0.0 to 2.1.0' AS ' '; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/oracle/036-HIVE-13354.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/036-HIVE-13354.oracle.sql b/metastore/scripts/upgrade/oracle/036-HIVE-13354.oracle.sql new file mode 100644 index 0000000..2f691b1 --- /dev/null +++ b/metastore/scripts/upgrade/oracle/036-HIVE-13354.oracle.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES varchar(2048); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048); http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql index 199ff4c..89f6374 100644 --- a/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/hive-txn-schema-1.3.0.oracle.sql @@ -83,6 +83,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START NUMBER(19), CQ_RUN_AS varchar(128), @@ -103,6 +104,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START NUMBER(19), CC_END NUMBER(19), http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/oracle/hive-txn-schema-2.1.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/hive-txn-schema-2.1.0.oracle.sql b/metastore/scripts/upgrade/oracle/hive-txn-schema-2.1.0.oracle.sql index d39baab..57d3abd 100644 --- a/metastore/scripts/upgrade/oracle/hive-txn-schema-2.1.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/hive-txn-schema-2.1.0.oracle.sql @@ -83,6 +83,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START NUMBER(19), CQ_RUN_AS varchar(128), @@ -103,6 +104,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START NUMBER(19), CC_END NUMBER(19), http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql index 5939b34..e6d5726 100644 --- a/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/upgrade-1.2.0-to-1.3.0.oracle.sql @@ -12,6 +12,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual; @031-HIVE-12381.oracle.sql; @032-HIVE-12832.oracle.sql; @035-HIVE-13395.oracle.sql; +@036-HIVE-13354.oracle.sql; UPDATE VERSION SET SCHEMA_VERSION='1.3.0', VERSION_COMMENT='Hive release version 1.3.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0' AS Status from dual; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/oracle/upgrade-2.0.0-to-2.1.0.oracle.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/oracle/upgrade-2.0.0-to-2.1.0.oracle.sql b/metastore/scripts/upgrade/oracle/upgrade-2.0.0-to-2.1.0.oracle.sql index e27047e..13bdcdd 100644 --- a/metastore/scripts/upgrade/oracle/upgrade-2.0.0-to-2.1.0.oracle.sql +++ b/metastore/scripts/upgrade/oracle/upgrade-2.0.0-to-2.1.0.oracle.sql @@ -2,6 +2,7 @@ SELECT 'Upgrading MetaStore schema from 2.0.0 to 2.1.0' AS Status from dual; @034-HIVE-13076.oracle.sql; @035-HIVE-13395.oracle.sql; +@036-HIVE-13354.oracle.sql; UPDATE VERSION SET SCHEMA_VERSION='2.1.0', VERSION_COMMENT='Hive release version 2.1.0' where VER_ID=1; SELECT 'Finished upgrading MetaStore schema from 2.0.0 to 2.1.0' AS Status from dual; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/postgres/035-HIVE-13354.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/035-HIVE-13354.postgres.sql b/metastore/scripts/upgrade/postgres/035-HIVE-13354.postgres.sql new file mode 100644 index 0000000..2f691b1 --- /dev/null +++ b/metastore/scripts/upgrade/postgres/035-HIVE-13354.postgres.sql @@ -0,0 +1,2 @@ +ALTER TABLE COMPACTION_QUEUE ADD CQ_TBLPROPERTIES varchar(2048); +ALTER TABLE COMPLETED_COMPACTIONS ADD CC_TBLPROPERTIES varchar(2048); http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql index b606f81..f998d18 100644 --- a/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/hive-txn-schema-1.3.0.postgres.sql @@ -83,6 +83,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -103,6 +104,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/postgres/hive-txn-schema-2.1.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/hive-txn-schema-2.1.0.postgres.sql b/metastore/scripts/upgrade/postgres/hive-txn-schema-2.1.0.postgres.sql index 262b93e..63ce2ee 100644 --- a/metastore/scripts/upgrade/postgres/hive-txn-schema-2.1.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/hive-txn-schema-2.1.0.postgres.sql @@ -83,6 +83,7 @@ CREATE TABLE COMPACTION_QUEUE ( CQ_PARTITION varchar(767), CQ_STATE char(1) NOT NULL, CQ_TYPE char(1) NOT NULL, + CQ_TBLPROPERTIES varchar(2048), CQ_WORKER_ID varchar(128), CQ_START bigint, CQ_RUN_AS varchar(128), @@ -103,6 +104,7 @@ CREATE TABLE COMPLETED_COMPACTIONS ( CC_PARTITION varchar(767), CC_STATE char(1) NOT NULL, CC_TYPE char(1) NOT NULL, + CC_TBLPROPERTIES varchar(2048), CC_WORKER_ID varchar(128), CC_START bigint, CC_END bigint, http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql index b1bcac0..4256825 100644 --- a/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/upgrade-1.2.0-to-1.3.0.postgres.sql @@ -12,6 +12,7 @@ SELECT 'Upgrading MetaStore schema from 1.2.0 to 1.3.0'; \i 030-HIVE-12831.postgres.sql; \i 031-HIVE-12832.postgres.sql; \i 034-HIVE-13395.postgres.sql; +\i 035-HIVE-13354.postgres.sql; UPDATE "VERSION" SET "SCHEMA_VERSION"='1.3.0', "VERSION_COMMENT"='Hive release version 1.3.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 1.2.0 to 1.3.0'; http://git-wip-us.apache.org/repos/asf/hive/blob/c57a5961/metastore/scripts/upgrade/postgres/upgrade-2.0.0-to-2.1.0.postgres.sql ---------------------------------------------------------------------- diff --git a/metastore/scripts/upgrade/postgres/upgrade-2.0.0-to-2.1.0.postgres.sql b/metastore/scripts/upgrade/postgres/upgrade-2.0.0-to-2.1.0.postgres.sql index a7293f7..fa3e723 100644 --- a/metastore/scripts/upgrade/postgres/upgrade-2.0.0-to-2.1.0.postgres.sql +++ b/metastore/scripts/upgrade/postgres/upgrade-2.0.0-to-2.1.0.postgres.sql @@ -2,6 +2,7 @@ SELECT 'Upgrading MetaStore schema from 2.0.0 to 2.1.0'; \i 033-HIVE-13076.postgres.sql; \i 034-HIVE-13395.postgres.sql; +\i 035-HIVE-13354.postgres.sql; UPDATE "VERSION" SET "SCHEMA_VERSION"='2.1.0', "VERSION_COMMENT"='Hive release version 2.1.0' where "VER_ID"=1; SELECT 'Finished upgrading MetaStore schema from 2.0.0 to 2.1.0';