This is an automated email from the ASF dual-hosted git repository. pvary pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new f2fbe72 HIVE-26044: Remove hardcoded version references from the tests (Peter Vary reviewed by Marton Bod and Stamatis Zampetakis) (#3115) f2fbe72 is described below commit f2fbe72ab524f603d89e867b2831087c93805256 Author: pvary <pv...@cloudera.com> AuthorDate: Tue Mar 22 12:20:52 2022 +0100 HIVE-26044: Remove hardcoded version references from the tests (Peter Vary reviewed by Marton Bod and Stamatis Zampetakis) (#3115) --- .../InformationSchemaWithPrivilegeTestBase.java | 6 ++++- .../hadoop/hive/ql/qoption/QTestSysDbHandler.java | 5 ++++- .../metastore/txn/TestCompactionTxnHandler.java | 3 ++- .../hive/ql/txn/compactor/CompactorTest.java | 3 ++- .../hadoop/hive/ql/txn/compactor/TestCleaner.java | 2 +- .../ql/txn/compactor/TestCompactionMetrics.java | 26 +++++++++++----------- .../hive/ql/txn/compactor/TestInitiator.java | 8 +++---- .../hadoop/hive/ql/txn/compactor/TestWorker.java | 4 ++-- .../metastore/dbinstall/rules/PostgresTPCDS.java | 5 ++++- 9 files changed, 37 insertions(+), 25 deletions(-) diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java index e0f8947..9573e50 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/server/InformationSchemaWithPrivilegeTestBase.java @@ -26,6 +26,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hive.testutils.MiniZooKeeperCluster; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; @@ -183,6 +184,7 @@ public abstract class InformationSchemaWithPrivilegeTestBase { private static MiniHS2 miniHS2 = null; private static MiniZooKeeperCluster zkCluster = null; private static Map<String, String> confOverlay; + private static String hiveSchemaVer; public static void setupInternal(boolean zookeeperSSLEnabled) throws Exception { @@ -228,6 +230,8 @@ public abstract class InformationSchemaWithPrivilegeTestBase { confOverlay.put(ConfVars.HIVE_ZOOKEEPER_SSL_ENABLE.varname, "true"); } miniHS2.start(confOverlay); + + hiveSchemaVer = MetaStoreSchemaInfoFactory.get(miniHS2.getServerConf()).getHiveSchemaVersion(); } @AfterClass @@ -287,7 +291,7 @@ public abstract class InformationSchemaWithPrivilegeTestBase { List<String> args = new ArrayList<String>(baseArgs); args.add("-f"); - args.add("../../metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-1.hive.sql"); + args.add("../../metastore/scripts/upgrade/hive/hive-schema-" + hiveSchemaVer + ".hive.sql"); BeeLine beeLine = new BeeLine(); int result = beeLine.begin(args.toArray(new String[] {}), null); beeLine.close(); diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java index 4834ad2..1ffacb6 100644 --- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java +++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/qoption/QTestSysDbHandler.java @@ -18,6 +18,7 @@ package org.apache.hadoop.hive.ql.qoption; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hadoop.hive.ql.QTestUtil; import org.apache.hive.testutils.HiveTestEnvSetup; import org.slf4j.Logger; @@ -44,7 +45,9 @@ public class QTestSysDbHandler implements QTestOptionHandler { @Override public void beforeTest(QTestUtil qt) throws Exception { if (enabled) { - String stsdbPath = HiveTestEnvSetup.HIVE_ROOT + "/metastore/scripts/upgrade/hive/hive-schema-4.0.0-alpha-1.hive.sql"; + String schemaVersion = MetaStoreSchemaInfoFactory.get(qt.getConf()).getHiveSchemaVersion(); + String stsdbPath = + HiveTestEnvSetup.HIVE_ROOT + "/metastore/scripts/upgrade/hive/hive-schema-" + schemaVersion + ".hive.sql"; qt.getCliDriver().processLine("source " + stsdbPath); qt.getCliDriver().processLine("use default"); } diff --git a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java index b9d34d5..46879b0 100644 --- a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java +++ b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java @@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.api.ShowCompactResponseElement; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.utils.TestTxnDbUtil; +import org.apache.hive.common.util.HiveVersionInfo; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -73,7 +74,7 @@ import static org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars.COMPA */ public class TestCompactionTxnHandler { - public static final String WORKER_VERSION = "4.0.0-alpha-1"; + public static final String WORKER_VERSION = HiveVersionInfo.getShortVersion(); private HiveConf conf = new HiveConf(); private TxnStore txnHandler; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java index a67795a..98a042a 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/CompactorTest.java @@ -77,6 +77,7 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.RecordWriter; import org.apache.hadoop.mapred.Reporter; import org.apache.hadoop.util.Progressable; +import org.apache.hive.common.util.HiveVersionInfo; import org.apache.thrift.TException; import org.junit.Before; import org.slf4j.Logger; @@ -112,7 +113,7 @@ import javax.management.ObjectName; public abstract class CompactorTest { static final private String CLASS_NAME = CompactorTest.class.getName(); static final private Logger LOG = LoggerFactory.getLogger(CLASS_NAME); - public static final String WORKER_VERSION = "4.0.0-alpha-1"; + public static final String WORKER_VERSION = HiveVersionInfo.getShortVersion(); protected TxnStore txnHandler; protected IMetaStoreClient ms; diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java index 757b01a..79b8348 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCleaner.java @@ -280,7 +280,7 @@ public class TestCleaner extends CompactorTest { FindNextCompactRequest findNextCompactRequest = new FindNextCompactRequest(); findNextCompactRequest.setWorkerId("fred"); - findNextCompactRequest.setWorkerVersion("4.0.0-alpha-1"); + findNextCompactRequest.setWorkerVersion(WORKER_VERSION); CompactionInfo ci = txnHandler.findNextToCompact(findNextCompactRequest); ci.runAs = System.getProperty("user.name"); long compactTxn = openTxn(TxnType.COMPACTION); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactionMetrics.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactionMetrics.java index a431955..45a595b 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactionMetrics.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestCompactionMetrics.java @@ -430,7 +430,7 @@ public class TestCompactionMetrics extends CompactorTest { elements.add(generateElement(4,"db", "tb3", "p1", CompactionType.MINOR, TxnStore.FAILED_RESPONSE)); elements.add(generateElement(6,"db1", "tb", null, CompactionType.MINOR, TxnStore.FAILED_RESPONSE, - System.currentTimeMillis(), true, "4.0.0-alpha-1", "4.0.0-alpha-1", 10)); + System.currentTimeMillis(), true, WORKER_VERSION, WORKER_VERSION, 10)); elements.add(generateElement(7,"db1", "tb2", null, CompactionType.MINOR, TxnStore.FAILED_RESPONSE)); elements.add(generateElement(8,"db1", "tb3", null, CompactionType.MINOR, TxnStore.FAILED_RESPONSE)); @@ -444,10 +444,10 @@ public class TestCompactionMetrics extends CompactorTest { elements.add(generateElement(14,"db3", "tb4", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE, System.currentTimeMillis(), false, null, null,20)); elements.add(generateElement(15,"db3", "tb5", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE, - System.currentTimeMillis(),true, "4.0.0-alpha-1", "4.0.0-alpha-1", 30)); + System.currentTimeMillis(),true, WORKER_VERSION, WORKER_VERSION, 30)); elements.add(generateElement(16,"db3", "tb6", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE)); elements.add(generateElement(17,"db3", "tb7", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE, - System.currentTimeMillis(),true, "4.0.0-alpha-1", "4.0.0-alpha-1",40)); + System.currentTimeMillis(),true, WORKER_VERSION, WORKER_VERSION,40)); scr.setCompacts(elements); AcidMetricService.updateMetricsFromShowCompact(scr); @@ -520,7 +520,7 @@ public class TestCompactionMetrics extends CompactorTest { long start = System.currentTimeMillis() - 1000L; List<ShowCompactResponseElement> elements = ImmutableList.of( generateElement(17, "db3", "tb7", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE, - System.currentTimeMillis(), true, "4.0.0-alpha-1", "4.0.0-alpha-1", start) + System.currentTimeMillis(), true, WORKER_VERSION, WORKER_VERSION, start) ); scr.setCompacts(elements); @@ -540,7 +540,7 @@ public class TestCompactionMetrics extends CompactorTest { long start = System.currentTimeMillis() - 1000L; List<ShowCompactResponseElement> elements = ImmutableList.of( generateElement(19, "db3", "tb7", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, - System.currentTimeMillis(), true, "4.0.0-alpha-1", "4.0.0-alpha-1", -1L, start) + System.currentTimeMillis(), true, WORKER_VERSION, WORKER_VERSION, -1L, start) ); scr.setCompacts(elements); @@ -591,9 +591,9 @@ public class TestCompactionMetrics extends CompactorTest { List<ShowCompactResponseElement> elements = ImmutableList.of( generateElement(15, "db3", "tb5", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE, - start, false, "4.0.0-alpha-1", "4.0.0-alpha-1", start - 1_000L), + start, false, WORKER_VERSION, WORKER_VERSION, start - 1_000L), generateElement(16, "db3", "tb6", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE, - start, false, "4.0.0-alpha-1", "4.0.0-alpha-1", start - 15_000L) + start, false, WORKER_VERSION, WORKER_VERSION, start - 15_000L) ); scr.setCompacts(elements); @@ -604,9 +604,9 @@ public class TestCompactionMetrics extends CompactorTest { // Check the reverse order elements = ImmutableList.of( generateElement(16, "db3", "tb6", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE, - start, false, "4.0.0-alpha-1", "4.0.0-alpha-1", start - 25_000L), + start, false, WORKER_VERSION, WORKER_VERSION, start - 25_000L), generateElement(15, "db3", "tb5", null, CompactionType.MINOR, TxnStore.WORKING_RESPONSE, - start, false, "4.0.0-alpha-1", "4.0.0-alpha-1", start - 1_000L) + start, false, WORKER_VERSION, WORKER_VERSION, start - 1_000L) ); scr.setCompacts(elements); AcidMetricService.updateMetricsFromShowCompact(scr); @@ -622,9 +622,9 @@ public class TestCompactionMetrics extends CompactorTest { List<ShowCompactResponseElement> elements = ImmutableList.of( generateElement(15, "db3", "tb5", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, - start, false, "4.0.0-alpha-1", "4.0.0-alpha-1", -1L, start - 1_000L), + start, false, WORKER_VERSION, WORKER_VERSION, -1L, start - 1_000L), generateElement(16, "db3", "tb6", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, - start, false, "4.0.0-alpha-1", "4.0.0-alpha-1", -1L, start - 15_000L) + start, false, WORKER_VERSION, WORKER_VERSION, -1L, start - 15_000L) ); scr.setCompacts(elements); @@ -635,9 +635,9 @@ public class TestCompactionMetrics extends CompactorTest { // Check the reverse order elements = ImmutableList.of( generateElement(16, "db3", "tb6", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, - start, false, "4.0.0-alpha-1", "4.0.0-alpha-1", -1L, start - 25_000L), + start, false, WORKER_VERSION, WORKER_VERSION, -1L, start - 25_000L), generateElement(15, "db3", "tb5", null, CompactionType.MINOR, TxnStore.CLEANING_RESPONSE, - start, false, "4.0.0-alpha-1", "4.0.0-alpha-1", -1L, start - 1_000L) + start, false, WORKER_VERSION, WORKER_VERSION, -1L, start - 1_000L) ); scr.setCompacts(elements); AcidMetricService.updateMetricsFromShowCompact(scr); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java index 084fb26..6a9a37d 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestInitiator.java @@ -82,8 +82,8 @@ public class TestInitiator extends CompactorTest { rqst = new CompactionRequest("default", "rflw2", CompactionType.MINOR); txnHandler.compact(rqst); - txnHandler.findNextToCompact(aFindNextCompactRequest(ServerUtils.hostname() + "-193892", "4.0.0-alpha-1")); - txnHandler.findNextToCompact(aFindNextCompactRequest("nosuchhost-193892", "4.0.0-alpha-1")); + txnHandler.findNextToCompact(aFindNextCompactRequest(ServerUtils.hostname() + "-193892", WORKER_VERSION)); + txnHandler.findNextToCompact(aFindNextCompactRequest("nosuchhost-193892", WORKER_VERSION)); startInitiator(); @@ -109,7 +109,7 @@ public class TestInitiator extends CompactorTest { CompactionRequest rqst = new CompactionRequest("default", "rfrw1", CompactionType.MINOR); txnHandler.compact(rqst); - txnHandler.findNextToCompact(aFindNextCompactRequest("nosuchhost-193892", "4.0.0-alpha-1")); + txnHandler.findNextToCompact(aFindNextCompactRequest("nosuchhost-193892", WORKER_VERSION)); conf.setTimeVar(HiveConf.ConfVars.HIVE_COMPACTOR_WORKER_TIMEOUT, 1L, TimeUnit.MILLISECONDS); @@ -1051,7 +1051,7 @@ public class TestInitiator extends CompactorTest { // need to mock the runtime version, because the manifest file won't be there in the mvn test setup Initiator initiator = Mockito.spy(new Initiator()); initiator.setConf(conf); - String runtimeVersion = "4.0.0-alpha-1-SNAPSHOT"; + String runtimeVersion = WORKER_VERSION; doReturn(runtimeVersion).when(initiator).getRuntimeVersion(); initiator.init(new AtomicBoolean(true)); initiator.run(); diff --git a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java index 996d956..1467e0c 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/txn/compactor/TestWorker.java @@ -988,13 +988,13 @@ public class TestWorker extends CompactorTest { burnThroughTransactions("default", "mtwb", 25); CompactionRequest rqst = new CompactionRequest("default", "mtwb", CompactionType.MINOR); - String initiatorVersion = "4.0.0-alpha-1"; + String initiatorVersion = "INITIATOR_VERSION"; rqst.setInitiatorVersion(initiatorVersion); txnHandler.compact(rqst); Worker worker = Mockito.spy(new Worker()); worker.setConf(conf); - String workerVersion = "4.0.0-alpha-1-SNAPSHOT"; + String workerVersion = "WORKER_VERSION"; doReturn(workerVersion).when(worker).getRuntimeVersion(); worker.init(new AtomicBoolean(true)); worker.run(); diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/PostgresTPCDS.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/PostgresTPCDS.java index 738e007..c1bfb7a 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/PostgresTPCDS.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/dbinstall/rules/PostgresTPCDS.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.hive.metastore.dbinstall.rules; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.MetaStoreSchemaInfoFactory; import org.apache.hadoop.hive.metastore.tools.schematool.MetastoreSchemaTool; import java.io.IOException; @@ -50,8 +52,9 @@ public class PostgresTPCDS extends Postgres { @Override public void install() { // Upgrade the metastore to latest by running explicitly a script. + String hiveSchemaVer = MetaStoreSchemaInfoFactory.get(new Configuration()).getHiveSchemaVersion(); try (InputStream script = PostgresTPCDS.class.getClassLoader() - .getResourceAsStream("sql/postgres/upgrade-3.1.3000-to-4.0.0-alpha-1.postgres.sql")) { + .getResourceAsStream("sql/postgres/upgrade-3.1.3000-to-" + hiveSchemaVer + ".postgres.sql")) { new MetastoreSchemaTool().runScript( buildArray( "-upgradeSchema",