HIVE-17541: Move testing related methods from MetaStoreUtils to some testing related utility (Zoltan Haindrich, reviewed by Alan Gates)
Signed-off-by: Zoltan Haindrich <k...@rxd.hu> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/229a7cc8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/229a7cc8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/229a7cc8 Branch: refs/heads/master Commit: 229a7cc819e2db04492591e4e30e15bb60c5a5a1 Parents: 6f5c113 Author: Zoltan Haindrich <k...@rxd.hu> Authored: Thu Sep 21 10:22:12 2017 +0200 Committer: Zoltan Haindrich <k...@rxd.hu> Committed: Thu Sep 21 10:22:18 2017 +0200 ---------------------------------------------------------------------- hcatalog/core/pom.xml | 7 + .../apache/hive/hcatalog/cli/TestPermsGrp.java | 5 +- .../mapreduce/TestHCatMultiOutputFormat.java | 4 +- .../mapreduce/TestHCatPartitionPublish.java | 5 +- hcatalog/webhcat/java-client/pom.xml | 7 + .../hive/hcatalog/api/TestHCatClient.java | 6 +- hcatalog/webhcat/svr/pom.xml | 7 + .../hive/hcatalog/templeton/TestWebHCatE2e.java | 4 +- .../security/TestHadoopAuthBridge23.java | 4 +- .../AbstractTestAuthorizationApiAuthorizer.java | 4 +- .../hadoop/hive/metastore/TestFilterHooks.java | 4 +- ...TestHiveMetaStoreWithEnvironmentContext.java | 4 +- .../hive/metastore/TestMarkPartitionRemote.java | 2 +- .../metastore/TestMetaStoreAuthorization.java | 2 +- .../TestMetaStoreEndFunctionListener.java | 4 +- .../metastore/TestMetaStoreEventListener.java | 4 +- .../TestMetaStoreEventListenerOnlyOnCommit.java | 4 +- .../metastore/TestMetaStoreInitListener.java | 4 +- .../metastore/TestMetaStoreListenersError.java | 4 +- .../hive/metastore/TestMetaStoreMetrics.java | 4 +- .../hive/metastore/TestRemoteHiveMetaStore.java | 4 +- .../TestRemoteHiveMetaStoreIpAddress.java | 4 +- .../hive/metastore/TestRetryingHMSHandler.java | 4 +- ...estDDLWithRemoteMetastoreSecondNamenode.java | 6 +- .../hive/ql/parse/TestReplicationScenarios.java | 6 +- .../hadoop/hive/ql/parse/WarehouseInstance.java | 4 +- .../security/StorageBasedMetastoreTestBase.java | 6 +- .../TestAuthorizationPreEventListener.java | 6 +- .../TestClientSideAuthorizationProvider.java | 6 +- .../TestMetastoreAuthorizationProvider.java | 6 +- .../TestMultiAuthorizationPreEventListener.java | 6 +- .../thrift/ThriftCliServiceMessageSizeTest.java | 4 +- itests/util/pom.xml | 6 + .../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 12 +- llap-server/pom.xml | 8 + .../services/impl/TestLlapWebServices.java | 4 +- metastore/pom.xml | 11 ++ .../hadoop/hive/metastore/MetaStoreUtils.java | 155 ++----------------- .../hive/metastore/MetaStoreTestUtils.java | 147 ++++++++++++++++++ .../metastore/TestHiveMetaStoreGetMetaConf.java | 2 +- .../TestHiveMetaStorePartitionSpecs.java | 2 +- .../TestRetriesInRetryingHMSHandler.java | 2 +- ql/pom.xml | 21 ++- .../TestHostAffinitySplitLocationProvider.java | 8 +- .../hadoop/hive/ql/metadata/TestHiveRemote.java | 4 +- service/pom.xml | 7 + .../cli/thrift/ThriftCLIServiceTest.java | 4 +- .../thrift/ThriftCliServiceTestWithCookie.java | 4 +- .../hive/service/server/TestHS2HttpServer.java | 4 +- 49 files changed, 320 insertions(+), 232 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/hcatalog/core/pom.xml ---------------------------------------------------------------------- diff --git a/hcatalog/core/pom.xml b/hcatalog/core/pom.xml index d151374..94e9fbe 100644 --- a/hcatalog/core/pom.xml +++ b/hcatalog/core/pom.xml @@ -67,6 +67,13 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> <!-- inter-project --> <dependency> <groupId>com.google.guava</groupId> http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java index 374c1d2..80c5d63 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestPermsGrp.java @@ -31,6 +31,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; @@ -80,8 +81,8 @@ public class TestPermsGrp extends TestCase { } - msPort = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); + msPort = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); isServerRunning = true; http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index 180e802..ba42ffd 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -34,7 +34,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -161,7 +161,7 @@ public class TestHCatMultiOutputFormat { metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString()); // Run hive metastore server - msPort = MetaStoreUtils.startMetaStore(metastoreConf); + msPort = MetaStoreTestUtils.startMetaStore(metastoreConf); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 Configuration conf = new Configuration(true); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java index 61b2f41..a000642 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatPartitionPublish.java @@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -104,9 +105,9 @@ public class TestHCatPartitionPublish { return; } - msPort = MetaStoreUtils.findFreePort(); + msPort = MetaStoreTestUtils.findFreePort(); - MetaStoreUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStore(msPort, HadoopThriftAuthBridge.getBridge()); Thread.sleep(10000); isServerRunning = true; securityManager = System.getSecurityManager(); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/hcatalog/webhcat/java-client/pom.xml ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/java-client/pom.xml b/hcatalog/webhcat/java-client/pom.xml index 9726df1..0c9fc15 100644 --- a/hcatalog/webhcat/java-client/pom.xml +++ b/hcatalog/webhcat/java-client/pom.xml @@ -75,6 +75,13 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.apache.hive.hcatalog</groupId> <artifactId>hive-hcatalog-core</artifactId> <version>${project.version}</version> http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index 0844b8e..d2474cc 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -35,7 +35,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; @@ -111,7 +111,7 @@ public class TestHCatClient { System.setProperty(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, DbNotificationListener.class.getName()); // turn on db notification listener on metastore - msPort = MetaStoreUtils.startMetaStore(); + msPort = MetaStoreTestUtils.startMetaStore(); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" @@ -798,7 +798,7 @@ public class TestHCatClient { HiveConf conf = new HiveConf(); conf.set("javax.jdo.option.ConnectionURL", hcatConf.get("javax.jdo.option.ConnectionURL") .replace("metastore", "target_metastore")); - replicationTargetHCatPort = MetaStoreUtils.startMetaStore(conf); + replicationTargetHCatPort = MetaStoreTestUtils.startMetaStore(conf); replicationTargetHCatConf = new HiveConf(hcatConf); replicationTargetHCatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + replicationTargetHCatPort); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/hcatalog/webhcat/svr/pom.xml ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/pom.xml b/hcatalog/webhcat/svr/pom.xml index a55ffe9..6bceee1 100644 --- a/hcatalog/webhcat/svr/pom.xml +++ b/hcatalog/webhcat/svr/pom.xml @@ -196,6 +196,13 @@ </dependency> <!-- test inter-project --> <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <version>${junit.version}</version> http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java index b62716e..22d2cc6 100644 --- a/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java +++ b/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/TestWebHCatE2e.java @@ -25,7 +25,7 @@ import org.apache.commons.httpclient.methods.DeleteMethod; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PutMethod; import org.apache.commons.httpclient.methods.StringRequestEntity; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.ql.ErrorMsg; import org.codehaus.jackson.map.ObjectMapper; import org.codehaus.jackson.type.TypeReference; @@ -74,7 +74,7 @@ public class TestWebHCatE2e { int webhcatPort = 50111; try { //in case concurrent tests are running on the same machine - webhcatPort = MetaStoreUtils.findFreePort(); + webhcatPort = MetaStoreTestUtils.findFreePort(); } catch (IOException ex) { LOG.warn("Unable to find free port; using default: " + webhcatPort); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java index ef36040..14c4e42 100644 --- a/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java +++ b/itests/hive-unit-hadoop2/src/test/java/org/apache/hadoop/hive/metastore/security/TestHadoopAuthBridge23.java @@ -24,7 +24,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.io.Text; @@ -150,7 +150,7 @@ public class TestHadoopAuthBridge23 { System.setProperty(HiveConf.ConfVars.METASTORE_CLUSTER_DELEGATION_TOKEN_STORE_CLS.varname, MyTokenStore.class.getName()); conf = new HiveConf(TestHadoopAuthBridge23.class); - MetaStoreUtils.startMetaStore(port, new MyHadoopThriftAuthBridge23()); + MetaStoreTestUtils.startMetaStore(port, new MyHadoopThriftAuthBridge23()); } /** http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java index 5ea809f..ebdfab2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/AbstractTestAuthorizationApiAuthorizer.java @@ -57,8 +57,8 @@ public abstract class AbstractTestAuthorizationApiAuthorizer { hiveConf = new HiveConf(); if (isRemoteMetastoreMode) { - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); } hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java index 69efc98..6a2bbd2 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestFilterHooks.java @@ -174,9 +174,9 @@ public class TestFilterHooks { hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); hiveConf.setVar(ConfVars.METASTORE_FILTER_HOOK, DummyMetaStoreFilterHookImpl.class.getName()); UtilsForTest.setNewDerbyDbLocation(hiveConf, TestFilterHooks.class.getSimpleName()); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); SessionState.start(new CliSessionState(hiveConf)); msc = new HiveMetaStoreClient(hiveConf); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java index dee5d77..3fa4b1e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreWithEnvironmentContext.java @@ -73,8 +73,8 @@ public class TestHiveMetaStoreWithEnvironmentContext extends TestCase { System.setProperty("hive.metastore.event.listeners", DummyListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java index 239bad0..e3c9a4e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java @@ -25,7 +25,7 @@ public class TestMarkPartitionRemote extends TestMarkPartition { @Override protected void setUp() throws Exception { super.setUp(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore()); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreTestUtils.startMetaStore()); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); } } http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java index 6f0282c..1bf76b5 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreAuthorization.java @@ -75,7 +75,7 @@ public class TestMetaStoreAuthorization extends TestCase { public void testMetaStoreAuthorization() throws Exception { setup(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); HiveMetaStoreClient client = new HiveMetaStoreClient(conf); FileSystem fs = null; http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java index 4f559d0..27bc791 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEndFunctionListener.java @@ -48,8 +48,8 @@ public class TestMetaStoreEndFunctionListener extends TestCase { DummyPreListener.class.getName()); System.setProperty("hive.metastore.end.function.listeners", DummyEndFunctionListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java index 82fb8c7..42e476a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java @@ -94,11 +94,11 @@ public class TestMetaStoreEventListener extends TestCase { System.setProperty("hive.metastore.pre.event.listeners", DummyPreListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN, metaConfVal); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java index 3302937..d9da422 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreEventListenerOnlyOnCommit.java @@ -50,8 +50,8 @@ public class TestMetaStoreEventListenerOnlyOnCommit extends TestCase { System.setProperty(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL.varname, DummyRawStoreControlledCommit.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java index 09b5cd8..9578ef9 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreInitListener.java @@ -41,8 +41,8 @@ public class TestMetaStoreInitListener extends TestCase { super.setUp(); System.setProperty("hive.metastore.init.hooks", DummyMetaStoreInitListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java index 5f86c26..b4d258c 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreListenersError.java @@ -33,7 +33,7 @@ public class TestMetaStoreListenersError extends TestCase { public void testInitListenerException() throws Throwable { System.setProperty("hive.metastore.init.hooks", ErrorInitListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); try { HiveMetaStore.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); } catch (Throwable throwable) { @@ -50,7 +50,7 @@ public class TestMetaStoreListenersError extends TestCase { System.setProperty("hive.metastore.init.hooks", ""); System.setProperty("hive.metastore.event.listeners", ErrorEventListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); try { HiveMetaStore.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); } catch (Throwable throwable) { http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java index e1cb1f3..a899739 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMetaStoreMetrics.java @@ -40,7 +40,7 @@ public class TestMetaStoreMetrics { @BeforeClass public static void before() throws Exception { - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); hiveConf = new HiveConf(TestMetaStoreMetrics.class); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); @@ -52,7 +52,7 @@ public class TestMetaStoreMetrics { "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); //Increments one HMS connection - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); //Increments one HMS connection (Hive.get()) SessionState.start(new CliSessionState(hiveConf)); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java index a8f9907..19bed9d 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStore.java @@ -42,9 +42,9 @@ public class TestRemoteHiveMetaStore extends TestHiveMetaStore { return; } - port = MetaStoreUtils.findFreePort(); + port = MetaStoreTestUtils.findFreePort(); System.out.println("Starting MetaStore Server on port " + port); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), hiveConf); isServerStarted = true; // This is default case with setugi off for both client and server http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java index 31580fc..449fe71 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRemoteHiveMetaStoreIpAddress.java @@ -48,11 +48,11 @@ public class TestRemoteHiveMetaStoreIpAddress extends TestCase { return; } - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); System.out.println("Starting MetaStore Server on port " + port); System.setProperty(ConfVars.METASTORE_EVENT_LISTENERS.varname, IpAddressListener.class.getName()); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); isServerStarted = true; // This is default case with setugi off for both client and server http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java index a7e5eb3..41494d1 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestRetryingHMSHandler.java @@ -53,8 +53,8 @@ public class TestRetryingHMSHandler extends TestCase { super.setUp(); System.setProperty("hive.metastore.pre.event.listeners", AlternateFailurePreListener.class.getName()); - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); hiveConf = new HiveConf(this.getClass()); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java index 36fa81e..4ce2731 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/TestDDLWithRemoteMetastoreSecondNamenode.java @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; @@ -80,8 +80,8 @@ public class TestDDLWithRemoteMetastoreSecondNamenode extends TestCase { SessionState.start(conf); // Test with remote metastore service - int port = MetaStoreUtils.findFreePort(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + int port = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); conf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); conf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); conf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java index d87a4c0..c67192b 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenarios.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore; import org.apache.hadoop.hive.metastore.InjectableBehaviourObjectStore.BehaviourInjection; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; @@ -133,7 +133,7 @@ public class TestReplicationScenarios { hconf.setBoolVar(HiveConf.ConfVars.REPLCMENABLED, true); hconf.setBoolVar(HiveConf.ConfVars.FIRE_EVENTS_FOR_DML, true); hconf.setVar(HiveConf.ConfVars.REPLCMDIR, TEST_PATH + "/cmroot/"); - msPort = MetaStoreUtils.startMetaStore(hconf); + msPort = MetaStoreTestUtils.startMetaStore(hconf); hconf.setVar(HiveConf.ConfVars.REPLDIR,TEST_PATH + "/hrepl/"); hconf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); @@ -158,7 +158,7 @@ public class TestReplicationScenarios { FileUtils.deleteDirectory(new File("metastore_db2")); HiveConf hconfMirrorServer = new HiveConf(); hconfMirrorServer.set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname, "jdbc:derby:;databaseName=metastore_db2;create=true"); - msPortMirror = MetaStoreUtils.startMetaStore(hconfMirrorServer); + msPortMirror = MetaStoreTestUtils.startMetaStore(hconfMirrorServer); hconfMirror = new HiveConf(hconf); hconfMirror.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPortMirror); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java index 19ad442..cde7a3e 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork; @@ -125,7 +125,7 @@ class WarehouseInstance implements Closeable { System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); - int metaStorePort = MetaStoreUtils.startMetaStore(hiveConf); + int metaStorePort = MetaStoreTestUtils.startMetaStore(hiveConf); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); Path testPath = new Path(hiveWarehouseLocation); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java index ccf835f..fa5ae80 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/StorageBasedMetastoreTestBase.java @@ -28,7 +28,7 @@ import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.ql.Driver; @@ -63,7 +63,7 @@ public class StorageBasedMetastoreTestBase { @Before public void setUp() throws Exception { - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, @@ -74,7 +74,7 @@ public class StorageBasedMetastoreTestBase { InjectableDummyAuthenticator.class.getName()); clientHiveConf = createHiveConf(); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), clientHiveConf); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge(), clientHiveConf); // Turn off client-side authorization clientHiveConf.setBoolVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_ENABLED,false); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java index 3b05731..102af6a 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestAuthorizationPreEventListener.java @@ -27,7 +27,7 @@ import junit.framework.TestCase; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -53,7 +53,7 @@ public class TestAuthorizationPreEventListener extends TestCase { super.setUp(); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -62,7 +62,7 @@ public class TestAuthorizationPreEventListener extends TestCase { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); clientHiveConf = new HiveConf(this.getClass()); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java index 24e49ba..60273e7 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestClientSideAuthorizationProvider.java @@ -26,7 +26,7 @@ import junit.framework.TestCase; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; @@ -58,13 +58,13 @@ public class TestClientSideAuthorizationProvider extends TestCase { super.setUp(); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); // Turn off metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, ""); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); clientHiveConf = new HiveConf(this.getClass()); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java index 8b90fa8..f0196c6 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMetastoreAuthorizationProvider.java @@ -29,7 +29,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; @@ -84,7 +84,7 @@ public class TestMetastoreAuthorizationProvider extends TestCase { super.setUp(); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); // Turn on metastore-side authorization System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, @@ -99,7 +99,7 @@ public class TestMetastoreAuthorizationProvider extends TestCase { System.setProperty(HiveConf.ConfVars.HIVE_AUTHORIZATION_TABLE_OWNER_GRANTS.varname, ""); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); clientHiveConf = createHiveConf(); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java index 3d9247d..62835b0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/TestMultiAuthorizationPreEventListener.java @@ -25,7 +25,7 @@ import java.util.List; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.ql.Driver; @@ -49,7 +49,7 @@ public class TestMultiAuthorizationPreEventListener { public static void setUp() throws Exception { - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); System.setProperty(HiveConf.ConfVars.METASTORE_PRE_EVENT_LISTENERS.varname, AuthorizationPreEventListener.class.getName()); @@ -62,7 +62,7 @@ public class TestMultiAuthorizationPreEventListener { System.setProperty(HiveConf.ConfVars.HIVE_METASTORE_AUTHENTICATOR_MANAGER.varname, HadoopDefaultMetastoreAuthenticator.class.getName()); - MetaStoreUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); + MetaStoreTestUtils.startMetaStore(port, HadoopThriftAuthBridge.getBridge()); clientHiveConf = new HiveConf(); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java index c859b8f..926efa3 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/service/cli/thrift/ThriftCliServiceMessageSizeTest.java @@ -28,7 +28,7 @@ import java.util.Map; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hive.service.Service; import org.apache.hive.service.auth.HiveAuthConstants; import org.apache.hive.service.auth.HiveAuthConstants.AuthTypes; @@ -55,7 +55,7 @@ public class ThriftCliServiceMessageSizeTest { @BeforeClass public static void setUpBeforeClass() throws Exception { // Find a free port - port = MetaStoreUtils.findFreePort(); + port = MetaStoreTestUtils.findFreePort(); hiveServer2 = new HiveServer2(); hiveConf = new HiveConf(); } http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/util/pom.xml ---------------------------------------------------------------------- diff --git a/itests/util/pom.xml b/itests/util/pom.xml index a44a55c..e6dc09f 100644 --- a/itests/util/pom.xml +++ b/itests/util/pom.xml @@ -103,6 +103,12 @@ </dependency> <dependency> <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> <artifactId>hive-llap-server</artifactId> <version>${project.version}</version> </dependency> http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java ---------------------------------------------------------------------- diff --git a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java index 06ddc22..5702de2 100644 --- a/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java +++ b/itests/util/src/main/java/org/apache/hive/jdbc/miniHS2/MiniHS2.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.llap.LlapItUtils; import org.apache.hadoop.hive.llap.daemon.MiniLlapCluster; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.util.ZooKeeperHiveHelper; @@ -198,9 +198,9 @@ public class MiniHS2 extends AbstractHiveService { super( hiveConf, "localhost", - (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT) : MetaStoreUtils + (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT) : MetaStoreTestUtils .findFreePort()), - (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT) : MetaStoreUtils + (usePortsFromConf ? hiveConf.getIntVar(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_HTTP_PORT) : MetaStoreTestUtils .findFreePort())); hiveConf.setLongVar(ConfVars.HIVE_SERVER2_MAX_START_ATTEMPTS, 3l); hiveConf.setTimeVar(ConfVars.HIVE_SERVER2_SLEEP_INTERVAL_BETWEEN_START_ATTEMPTS, 10, @@ -277,7 +277,7 @@ public class MiniHS2 extends AbstractHiveService { hiveConf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, metaStoreURL); if (!usePortsFromConf) { // reassign a new port, just in case if one of the MR services grabbed the last one - setBinaryPort(MetaStoreUtils.findFreePort()); + setBinaryPort(MetaStoreTestUtils.findFreePort()); } hiveConf.setVar(ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST, getHost()); hiveConf.setIntVar(ConfVars.HIVE_SERVER2_THRIFT_PORT, getBinaryPort()); @@ -310,9 +310,9 @@ public class MiniHS2 extends AbstractHiveService { public void start(Map<String, String> confOverlay) throws Exception { if (isMetastoreRemote) { - int metaStorePort = MetaStoreUtils.findFreePort(); + int metaStorePort = MetaStoreTestUtils.findFreePort(); getHiveConf().setVar(ConfVars.METASTOREURIS, "thrift://localhost:" + metaStorePort); - MetaStoreUtils.startMetaStore(metaStorePort, HadoopThriftAuthBridge.getBridge(), getHiveConf()); + MetaStoreTestUtils.startMetaStore(metaStorePort, HadoopThriftAuthBridge.getBridge(), getHiveConf()); } hiveServer2 = new HiveServer2(); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/llap-server/pom.xml ---------------------------------------------------------------------- diff --git a/llap-server/pom.xml b/llap-server/pom.xml index ff62dc3..47a04cc 100644 --- a/llap-server/pom.xml +++ b/llap-server/pom.xml @@ -235,6 +235,14 @@ <artifactId>jettison</artifactId> </dependency> + <!-- test intra-project --> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> <!-- test inter-project --> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/services/impl/TestLlapWebServices.java ---------------------------------------------------------------------- diff --git a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/services/impl/TestLlapWebServices.java b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/services/impl/TestLlapWebServices.java index 833a8f5..698a56e 100644 --- a/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/services/impl/TestLlapWebServices.java +++ b/llap-server/src/test/org/apache/hadoop/hive/llap/daemon/services/impl/TestLlapWebServices.java @@ -16,7 +16,7 @@ package org.apache.hadoop.hive.llap.daemon.services.impl; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -34,7 +34,7 @@ public class TestLlapWebServices { @BeforeClass public static void beforeTests() throws Exception { - llapWSPort = MetaStoreUtils.findFreePortExcepting( + llapWSPort = MetaStoreTestUtils.findFreePortExcepting( Integer.valueOf(HiveConf.ConfVars.LLAP_DAEMON_WEB_PORT.getDefaultValue())); llapWS = new LlapWebServices(llapWSPort, null, null); llapWS.init(new HiveConf()); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/metastore/pom.xml ---------------------------------------------------------------------- diff --git a/metastore/pom.xml b/metastore/pom.xml index 8c35649..5430580 100644 --- a/metastore/pom.xml +++ b/metastore/pom.xml @@ -310,6 +310,17 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>test-jar</goal> + </goals> + </execution> + </executions> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index 6cf9a5c..b51446d 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -24,9 +24,6 @@ import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.math.BigDecimal; import java.math.BigInteger; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; import java.net.URL; import java.net.URLClassLoader; import java.nio.charset.Charset; @@ -65,7 +62,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.metastore.api.Decimal; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.SkewedInfo; -import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.shims.ShimLoader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -78,7 +74,6 @@ import org.apache.hadoop.fs.PathFilter; import org.apache.hadoop.hive.common.JavaUtils; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; @@ -118,7 +113,7 @@ import javax.annotation.Nullable; public class MetaStoreUtils { - protected static final Logger LOG = LoggerFactory.getLogger("hive.log"); + private static final Logger LOG = LoggerFactory.getLogger("hive.log"); // Right now we only support one special character '/'. // More special characters can be added accordingly in the future. @@ -704,7 +699,9 @@ public class MetaStoreUtils { * @return */ static public String validateColumnType(String type) { - if (type.equals(TYPE_FROM_DESERIALIZER)) return null; + if (type.equals(TYPE_FROM_DESERIALIZER)) { + return null; + } int last = 0; boolean lastAlphaDigit = isValidTypeChar(type.charAt(last)); for (int i = 1; i <= type.length(); i++) { @@ -1251,134 +1248,6 @@ public class MetaStoreUtils { } } - public static int startMetaStore() throws Exception { - return startMetaStore(HadoopThriftAuthBridge.getBridge(), null); - } - - public static int startMetaStore(final HadoopThriftAuthBridge bridge, HiveConf conf) throws Exception { - int port = findFreePort(); - startMetaStore(port, bridge, conf); - return port; - } - - public static int startMetaStore(HiveConf conf) throws Exception { - return startMetaStore(HadoopThriftAuthBridge.getBridge(), conf); - } - - public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception { - startMetaStore(port, bridge, null); - } - - public static void startMetaStore(final int port, - final HadoopThriftAuthBridge bridge, HiveConf hiveConf) - throws Exception{ - if (hiveConf == null) { - hiveConf = new HiveConf(HMSHandler.class); - } - final HiveConf finalHiveConf = hiveConf; - Thread thread = new Thread(new Runnable() { - @Override - public void run() { - try { - HiveMetaStore.startMetaStore(port, bridge, finalHiveConf); - } catch (Throwable e) { - LOG.error("Metastore Thrift Server threw an exception...",e); - } - } - }); - thread.setDaemon(true); - thread.start(); - loopUntilHMSReady(port); - } - - /** - * A simple connect test to make sure that the metastore is up - * @throws Exception - */ - private static void loopUntilHMSReady(int port) throws Exception { - int retries = 0; - Exception exc = null; - while (true) { - try { - Socket socket = new Socket(); - socket.connect(new InetSocketAddress(port), 5000); - socket.close(); - return; - } catch (Exception e) { - if (retries++ > 60) { //give up - exc = e; - break; - } - Thread.sleep(1000); - } - } - // something is preventing metastore from starting - // print the stack from all threads for debugging purposes - LOG.error("Unable to connect to metastore server: " + exc.getMessage()); - LOG.info("Printing all thread stack traces for debugging before throwing exception."); - LOG.info(getAllThreadStacksAsString()); - throw exc; - } - - private static String getAllThreadStacksAsString() { - Map<Thread, StackTraceElement[]> threadStacks = Thread.getAllStackTraces(); - StringBuilder sb = new StringBuilder(); - for (Map.Entry<Thread, StackTraceElement[]> entry : threadStacks.entrySet()) { - Thread t = entry.getKey(); - sb.append(System.lineSeparator()); - sb.append("Name: ").append(t.getName()).append(" State: ").append(t.getState()); - addStackString(entry.getValue(), sb); - } - return sb.toString(); - } - - private static void addStackString(StackTraceElement[] stackElems, StringBuilder sb) { - sb.append(System.lineSeparator()); - for (StackTraceElement stackElem : stackElems) { - sb.append(stackElem).append(System.lineSeparator()); - } - } - - /** - * Finds a free port on the machine. - * - * @return - * @throws IOException - */ - public static int findFreePort() throws IOException { - ServerSocket socket= new ServerSocket(0); - int port = socket.getLocalPort(); - socket.close(); - return port; - } - - /** - * Finds a free port on the machine, but allow the - * ability to specify a port number to not use, no matter what. - */ - public static int findFreePortExcepting(int portToExclude) throws IOException { - ServerSocket socket1 = null; - ServerSocket socket2 = null; - try { - socket1 = new ServerSocket(0); - socket2 = new ServerSocket(0); - if (socket1.getLocalPort() != portToExclude) { - return socket1.getLocalPort(); - } - // If we're here, then socket1.getLocalPort was the port to exclude - // Since both sockets were open together at a point in time, we're - // guaranteed that socket2.getLocalPort() is not the same. - return socket2.getLocalPort(); - } finally { - if (socket1 != null){ - socket1.close(); - } - if (socket2 != null){ - socket2.close(); - } - } - } - /** * Catches exceptions that can't be handled and bundles them to MetaException * @@ -2026,7 +1895,9 @@ public class MetaStoreUtils { for (FieldSchema fs : sd.getCols()) { md.update(fs.getName().getBytes(ENCODING)); md.update(fs.getType().getBytes(ENCODING)); - if (fs.getComment() != null) md.update(fs.getComment().getBytes(ENCODING)); + if (fs.getComment() != null) { + md.update(fs.getComment().getBytes(ENCODING)); + } } if (sd.getInputFormat() != null) { md.update(sd.getInputFormat().getBytes(ENCODING)); @@ -2054,7 +1925,9 @@ public class MetaStoreUtils { } if (sd.getBucketCols() != null) { List<String> bucketCols = new ArrayList<>(sd.getBucketCols()); - for (String bucket : bucketCols) md.update(bucket.getBytes(ENCODING)); + for (String bucket : bucketCols) { + md.update(bucket.getBytes(ENCODING)); + } } if (sd.getSortCols() != null) { SortedSet<Order> orders = new TreeSet<>(sd.getSortCols()); @@ -2067,7 +1940,9 @@ public class MetaStoreUtils { SkewedInfo skewed = sd.getSkewedInfo(); if (skewed.getSkewedColNames() != null) { SortedSet<String> colnames = new TreeSet<>(skewed.getSkewedColNames()); - for (String colname : colnames) md.update(colname.getBytes(ENCODING)); + for (String colname : colnames) { + md.update(colname.getBytes(ENCODING)); + } } if (skewed.getSkewedColValues() != null) { SortedSet<String> sortedOuterList = new TreeSet<>(); @@ -2075,7 +1950,9 @@ public class MetaStoreUtils { SortedSet<String> sortedInnerList = new TreeSet<>(innerList); sortedOuterList.add(StringUtils.join(sortedInnerList, ".")); } - for (String colval : sortedOuterList) md.update(colval.getBytes(ENCODING)); + for (String colval : sortedOuterList) { + md.update(colval.getBytes(ENCODING)); + } } if (skewed.getSkewedColValueLocationMaps() != null) { SortedMap<String, String> sortedMap = new TreeMap<>(); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java b/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java new file mode 100644 index 0000000..f88d0dd --- /dev/null +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/MetaStoreTestUtils.java @@ -0,0 +1,147 @@ +package org.apache.hadoop.hive.metastore; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.util.Map; + +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MetaStoreTestUtils { + + private static final Logger LOG = LoggerFactory.getLogger("hive.log"); + + public static int startMetaStore() throws Exception { + return MetaStoreTestUtils.startMetaStore(HadoopThriftAuthBridge.getBridge(), null); + } + + public static int startMetaStore(final HadoopThriftAuthBridge bridge, HiveConf conf) throws Exception { + int port = MetaStoreTestUtils.findFreePort(); + MetaStoreTestUtils.startMetaStore(port, bridge, conf); + return port; + } + + public static int startMetaStore(HiveConf conf) throws Exception { + return startMetaStore(HadoopThriftAuthBridge.getBridge(), conf); + } + + public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception { + MetaStoreTestUtils.startMetaStore(port, bridge, null); + } + + public static void startMetaStore(final int port, + final HadoopThriftAuthBridge bridge, HiveConf hiveConf) + throws Exception{ + if (hiveConf == null) { + hiveConf = new HiveConf(HMSHandler.class); + } + final HiveConf finalHiveConf = hiveConf; + Thread thread = new Thread(new Runnable() { + @Override + public void run() { + try { + HiveMetaStore.startMetaStore(port, bridge, finalHiveConf); + } catch (Throwable e) { + LOG.error("Metastore Thrift Server threw an exception...", e); + } + } + }); + thread.setDaemon(true); + thread.start(); + MetaStoreTestUtils.loopUntilHMSReady(port); + } + + /** + * A simple connect test to make sure that the metastore is up + * @throws Exception + */ + public static void loopUntilHMSReady(int port) throws Exception { + int retries = 0; + Exception exc = null; + while (true) { + try { + Socket socket = new Socket(); + socket.connect(new InetSocketAddress(port), 5000); + socket.close(); + return; + } catch (Exception e) { + if (retries++ > 60) { //give up + exc = e; + break; + } + Thread.sleep(1000); + } + } + // something is preventing metastore from starting + // print the stack from all threads for debugging purposes + LOG.error("Unable to connect to metastore server: " + exc.getMessage()); + LOG.info("Printing all thread stack traces for debugging before throwing exception."); + LOG.info(MetaStoreTestUtils.getAllThreadStacksAsString()); + throw exc; + } + + public static String getAllThreadStacksAsString() { + Map<Thread, StackTraceElement[]> threadStacks = Thread.getAllStackTraces(); + StringBuilder sb = new StringBuilder(); + for (Map.Entry<Thread, StackTraceElement[]> entry : threadStacks.entrySet()) { + Thread t = entry.getKey(); + sb.append(System.lineSeparator()); + sb.append("Name: ").append(t.getName()).append(" State: ").append(t.getState()); + MetaStoreTestUtils.addStackString(entry.getValue(), sb); + } + return sb.toString(); + } + + public static void addStackString(StackTraceElement[] stackElems, StringBuilder sb) { + sb.append(System.lineSeparator()); + for (StackTraceElement stackElem : stackElems) { + sb.append(stackElem).append(System.lineSeparator()); + } + } + + /** + * Finds a free port on the machine. + * + * @return + * @throws IOException + */ + public static int findFreePort() throws IOException { + ServerSocket socket= new ServerSocket(0); + int port = socket.getLocalPort(); + socket.close(); + return port; + } + + /** + * Finds a free port on the machine, but allow the + * ability to specify a port number to not use, no matter what. + */ + public static int findFreePortExcepting(int portToExclude) throws IOException { + ServerSocket socket1 = null; + ServerSocket socket2 = null; + try { + socket1 = new ServerSocket(0); + socket2 = new ServerSocket(0); + if (socket1.getLocalPort() != portToExclude) { + return socket1.getLocalPort(); + } + // If we're here, then socket1.getLocalPort was the port to exclude + // Since both sockets were open together at a point in time, we're + // guaranteed that socket2.getLocalPort() is not the same. + return socket2.getLocalPort(); + } finally { + if (socket1 != null){ + socket1.close(); + } + if (socket2 != null){ + socket2.close(); + } + } + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java index da883d2..f71911e 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java @@ -79,7 +79,7 @@ public class TestHiveMetaStoreGetMetaConf { metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname, MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); metastoreConf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL, false); - int msPort = MetaStoreUtils.startMetaStore(metastoreConf); + int msPort = MetaStoreTestUtils.startMetaStore(metastoreConf); hiveConf = new HiveConf(TestHiveMetaStoreGetMetaConf.class); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java index 61fe7e1..86462ff 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java @@ -89,7 +89,7 @@ public class TestHiveMetaStorePartitionSpecs { HiveConf metastoreConf = new HiveConf(); metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname, MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); - msPort = MetaStoreUtils.startMetaStore(metastoreConf); + msPort = MetaStoreTestUtils.startMetaStore(metastoreConf); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); hiveConf = new HiveConf(TestHiveMetaStorePartitionSpecs.class); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java index d77cc27..0d2a9cb 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestRetriesInRetryingHMSHandler.java @@ -38,7 +38,7 @@ public class TestRetriesInRetryingHMSHandler { @BeforeClass public static void setup() throws IOException { hiveConf = new HiveConf(); - int port = MetaStoreUtils.findFreePort(); + int port = MetaStoreTestUtils.findFreePort(); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); hiveConf.setIntVar(HiveConf.ConfVars.HMSHANDLERATTEMPTS, RETRY_ATTEMPTS); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/ql/pom.xml ---------------------------------------------------------------------- diff --git a/ql/pom.xml b/ql/pom.xml index 16b88d6..3dfb421 100644 --- a/ql/pom.xml +++ b/ql/pom.xml @@ -54,13 +54,6 @@ </dependency> <dependency> <groupId>org.apache.hive</groupId> - <artifactId>hive-common</artifactId> - <version>${project.version}</version> - <scope>test</scope> - <type>test-jar</type> - </dependency> - <dependency> - <groupId>org.apache.hive</groupId> <artifactId>hive-metastore</artifactId> <version>${project.version}</version> </dependency> @@ -460,6 +453,20 @@ <version>${opencsv.version}</version> </dependency> <!-- test intra-project --> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-common</artifactId> + <version>${project.version}</version> + <scope>test</scope> + <type>test-jar</type> + </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> <!-- test inter-project --> <dependency> <groupId>org.apache.parquet</groupId> http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHostAffinitySplitLocationProvider.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHostAffinitySplitLocationProvider.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHostAffinitySplitLocationProvider.java index 1e701c0..13f4676 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHostAffinitySplitLocationProvider.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/tez/TestHostAffinitySplitLocationProvider.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.math.stat.descriptive.SummaryStatistics; +import org.apache.commons.math3.stat.descriptive.SummaryStatistics; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.ql.io.orc.OrcSplit; import org.apache.hadoop.mapred.FileSplit; @@ -125,7 +125,9 @@ public class TestHostAffinitySplitLocationProvider { } lastLocations[splitIx] = splitLocation; } - if (locs == MIN_LOC_COUNT) continue; + if (locs == MIN_LOC_COUNT) { + continue; + } String msgTail = " when going to " + locs + " locations"; String movedMsg = moved + " splits moved", newMsg = newLoc + " splits went to the new node"; @@ -223,7 +225,7 @@ public class TestHostAffinitySplitLocationProvider { } // All of this is completely bogus and mostly captures the following function: // f(output) = I-eyeballed-the(output) == they-look-ok. - // It's pretty much a golden file... + // It's pretty much a golden file... // The fact that stdev doesn't increase with increasing missCount is captured outside. double avg = ss.getSum()/ss.getN(), stdev = ss.getStandardDeviation(), cv = stdev/avg; double allowedMin = avg - 2.5 * stdev, allowedMax = avg + 2.5 * stdev; http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java index 86263c0..d115e82 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.net.ServerSocket; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.util.StringUtils; /** @@ -43,7 +43,7 @@ public class TestHiveRemote extends TestHive { hiveConf .setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore()); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreTestUtils.startMetaStore()); try { hm = Hive.get(hiveConf); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/service/pom.xml ---------------------------------------------------------------------- diff --git a/service/pom.xml b/service/pom.xml index 8c53ed7..412bde5 100644 --- a/service/pom.xml +++ b/service/pom.xml @@ -250,6 +250,13 @@ <scope>test</scope> <type>test-jar</type> </dependency> + <dependency> + <groupId>org.apache.hive</groupId> + <artifactId>hive-metastore</artifactId> + <version>${project.version}</version> + <type>test-jar</type> + <scope>test</scope> + </dependency> <!-- test inter-project --> <dependency> <groupId>junit</groupId> http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java ---------------------------------------------------------------------- diff --git a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java index 4c59fca..5024f41 100644 --- a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java +++ b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCLIServiceTest.java @@ -25,7 +25,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hive.service.Service; import org.apache.hive.service.cli.OperationHandle; import org.apache.hive.service.cli.OperationState; @@ -60,7 +60,7 @@ public abstract class ThriftCLIServiceTest { @BeforeClass public static void setUpBeforeClass() throws Exception { // Find a free port - port = MetaStoreUtils.findFreePort(); + port = MetaStoreTestUtils.findFreePort(); hiveServer2 = new HiveServer2(); hiveConf = new HiveConf(); } http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java ---------------------------------------------------------------------- diff --git a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java index 13f582d..6660097 100644 --- a/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java +++ b/service/src/test/org/apache/hive/service/cli/thrift/ThriftCliServiceTestWithCookie.java @@ -29,7 +29,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hive.service.Service; import org.apache.hive.service.auth.HiveAuthConstants; import org.apache.hive.service.auth.HiveAuthConstants.AuthTypes; @@ -63,7 +63,7 @@ public class ThriftCliServiceTestWithCookie { @BeforeClass public static void setUpBeforeClass() throws Exception { // Find a free port - port = MetaStoreUtils.findFreePort(); + port = MetaStoreTestUtils.findFreePort(); hiveServer2 = new HiveServer2(); hiveConf = new HiveConf(); hiveConf.setBoolVar(ConfVars.HIVE_SERVER2_THRIFT_HTTP_COOKIE_AUTH_ENABLED, true); http://git-wip-us.apache.org/repos/asf/hive/blob/229a7cc8/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java ---------------------------------------------------------------------- diff --git a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java index 8a1f129..a4606d0 100644 --- a/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java +++ b/service/src/test/org/apache/hive/service/server/TestHS2HttpServer.java @@ -28,7 +28,7 @@ import java.net.URL; import org.apache.commons.io.IOUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; +import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -56,7 +56,7 @@ public class TestHS2HttpServer { @BeforeClass public static void beforeTests() throws Exception { - webUIPort = MetaStoreUtils.findFreePortExcepting( + webUIPort = MetaStoreTestUtils.findFreePortExcepting( Integer.valueOf(ConfVars.HIVE_SERVER2_WEBUI_PORT.getDefaultValue())); hiveConf = new HiveConf(); hiveConf.set(ConfVars.METASTOREPWD.varname, metastorePasswd);
