Repository: hive Updated Branches: refs/heads/master 882a7f000 -> 0ea9d35de
HIVE-13676 : Tests failing because metastore doesn't come up (Rajat Khandelwal via Ashutosh Chauhan) Signed-off-by: Ashutosh Chauhan <hashut...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0ea9d35d Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0ea9d35d Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0ea9d35d Branch: refs/heads/master Commit: 0ea9d35de39df94cb7c7ff4274c7ea4b177ba489 Parents: 882a7f0 Author: Rajat Khandelwal <pro...@apache.org> Authored: Tue May 10 01:32:00 2016 -0800 Committer: Ashutosh Chauhan <hashut...@apache.org> Committed: Tue May 10 09:47:31 2016 -0700 ---------------------------------------------------------------------- .../mapreduce/TestHCatMultiOutputFormat.java | 27 +++-------- .../hive/hcatalog/api/TestHCatClient.java | 49 ++++---------------- .../hive/metastore/TestMarkPartitionRemote.java | 26 +---------- .../hadoop/hive/metastore/MetaStoreUtils.java | 26 +++++++++-- .../metastore/TestHiveMetaStoreGetMetaConf.java | 23 ++------- .../TestHiveMetaStorePartitionSpecs.java | 22 ++------- .../hadoop/hive/ql/metadata/TestHiveRemote.java | 34 +------------- 7 files changed, 48 insertions(+), 159 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java ---------------------------------------------------------------------- diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java index 9fa263d..6ff48ee 100644 --- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java +++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/mapreduce/TestHCatMultiOutputFormat.java @@ -33,8 +33,8 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.SerDeInfo; @@ -89,7 +89,7 @@ public class TestHCatMultiOutputFormat { private static HiveConf hiveConf; private static File workDir; - private static final String msPort = "20199"; + private static int msPort; private static Thread t; static { @@ -98,22 +98,6 @@ public class TestHCatMultiOutputFormat { schemaMap.put(tableNames[2], new HCatSchema(ColumnHolder.hCattest3Cols)); } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - String warehouseConf = HiveConf.ConfVars.METASTOREWAREHOUSE.varname + "=" - + warehousedir.toString(); - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", warehouseConf}); - } catch (Throwable t) { - System.err.println("Exiting. Got exception from metastore: " + t.getMessage()); - t.printStackTrace(); - } - } - - } - /** * Private class which holds all the data for the test cases */ @@ -173,10 +157,11 @@ public class TestHCatMultiOutputFormat { warehousedir = new Path(System.getProperty("test.warehouse.dir")); - // Run hive metastore server - t = new Thread(new RunMS()); - t.start(); + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setVar(HiveConf.ConfVars.METASTOREWAREHOUSE, warehousedir.toString()); + // Run hive metastore server + msPort = MetaStoreUtils.startMetaStore(metastoreConf); // LocalJobRunner does not work with mapreduce OutputCommitter. So need // to use MiniMRCluster. MAPREDUCE-2350 Configuration conf = new Configuration(true); http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java ---------------------------------------------------------------------- diff --git a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java index aa9c7d3..48ee7cf 100644 --- a/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java +++ b/hcatalog/webhcat/java-client/src/test/java/org/apache/hive/hcatalog/api/TestHCatClient.java @@ -34,8 +34,8 @@ import com.google.common.collect.Iterables; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.IMetaStoreClient; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NotificationEvent; @@ -83,41 +83,14 @@ import javax.annotation.Nullable; public class TestHCatClient { private static final Logger LOG = LoggerFactory.getLogger(TestHCatClient.class); - private static final String msPort = "20101"; + private static int msPort; private static HiveConf hcatConf; private static boolean isReplicationTargetHCatRunning = false; - private static final String replicationTargetHCatPort = "20102"; + private static int replicationTargetHCatPort; private static HiveConf replicationTargetHCatConf; private static SecurityManager securityManager; private static boolean useExternalMS = false; - public static class RunMS implements Runnable { - - private final String msPort; - private List<String> args = new ArrayList<String>(); - - public RunMS(String msPort) { - this.msPort = msPort; - this.args.add("-v"); - this.args.add("-p"); - this.args.add(this.msPort); - } - - public RunMS arg(String arg) { - this.args.add(arg); - return this; - } - - @Override - public void run() { - try { - HiveMetaStore.main(args.toArray(new String[args.size()])); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } // class RunMS; - @AfterClass public static void tearDown() throws Exception { if (!useExternalMS) { @@ -142,10 +115,7 @@ public class TestHCatClient { System.setProperty(HiveConf.ConfVars.METASTORE_EVENT_LISTENERS.varname, DbNotificationListener.class.getName()); // turn on db notification listener on metastore - Thread t = new Thread(new RunMS(msPort)); - t.start(); - Thread.sleep(10000); - + msPort = MetaStoreUtils.startMetaStore(); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" @@ -832,13 +802,10 @@ public class TestHCatClient { private void startReplicationTargetMetaStoreIfRequired() throws Exception { if (!isReplicationTargetHCatRunning) { - Thread t = new Thread(new RunMS(replicationTargetHCatPort) - .arg("--hiveconf") - .arg("javax.jdo.option.ConnectionURL") // Reset, to use a different Derby instance. - .arg(hcatConf.get("javax.jdo.option.ConnectionURL") - .replace("metastore", "target_metastore"))); - t.start(); - Thread.sleep(10000); + HiveConf conf = new HiveConf(); + conf.set("javax.jdo.option.ConnectionURL", hcatConf.get("javax.jdo.option.ConnectionURL") + .replace("metastore", "target_metastore")); + replicationTargetHCatPort = MetaStoreUtils.startMetaStore(conf); replicationTargetHCatConf = new HiveConf(hcatConf); replicationTargetHCatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + replicationTargetHCatPort); http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java index 7576f39..239bad0 100644 --- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java +++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java @@ -19,37 +19,13 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.MetaStoreUtils; public class TestMarkPartitionRemote extends TestMarkPartition { - private static class RunMS implements Runnable { - - private final int port; - - public RunMS(int port) { - this.port = port; - } - @Override - public void run() { - try { - HiveMetaStore.main(new String[] { String.valueOf(port) }); - } catch (Throwable e) { - e.printStackTrace(System.err); - assert false; - } - } - - } @Override protected void setUp() throws Exception { super.setUp(); - int port = MetaStoreUtils.findFreePort(); - Thread t = new Thread(new RunMS(port)); - t.setDaemon(true); - t.start(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore()); hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTCONNECTIONRETRIES, 3); - Thread.sleep(30000); } } http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java index da3da8b..e01fe45 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java @@ -1176,22 +1176,38 @@ public class MetaStoreUtils { } catch (IOException e) { throw new MetaException("Unable to : " + path); } + } + public static int startMetaStore() throws Exception { + return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), null); } - public static void startMetaStore(final int port, - final HadoopThriftAuthBridge bridge) throws Exception { - startMetaStore(port, bridge, new HiveConf(HMSHandler.class)); + public static int startMetaStore(final HadoopThriftAuthBridge bridge, HiveConf conf) throws Exception { + int port = findFreePort(); + startMetaStore(port, bridge, conf); + return port; + } + + public static int startMetaStore(HiveConf conf) throws Exception { + return startMetaStore(ShimLoader.getHadoopThriftAuthBridge(), conf); + } + + public static void startMetaStore(final int port, final HadoopThriftAuthBridge bridge) throws Exception { + startMetaStore(port, bridge, null); } public static void startMetaStore(final int port, - final HadoopThriftAuthBridge bridge, final HiveConf hiveConf) + final HadoopThriftAuthBridge bridge, HiveConf hiveConf) throws Exception{ + if (hiveConf == null) { + hiveConf = new HiveConf(HMSHandler.class); + } + final HiveConf finalHiveConf = hiveConf; Thread thread = new Thread(new Runnable() { @Override public void run() { try { - HiveMetaStore.startMetaStore(port, bridge, hiveConf); + HiveMetaStore.startMetaStore(port, bridge, finalHiveConf); } catch (Throwable e) { LOG.error("Metastore Thrift Server threw an exception...",e); } http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java index 3f4561c..da883d2 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStoreGetMetaConf.java @@ -40,7 +40,6 @@ public class TestHiveMetaStoreGetMetaConf { public ExpectedException thrown = ExpectedException.none(); private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStoreGetMetaConf.class); - private static final String msPort = "20103"; private static HiveConf hiveConf; private static SecurityManager securityManager; @@ -65,20 +64,6 @@ public class TestHiveMetaStoreGetMetaConf { } } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", - "hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName(), - "--hiveconf", "hive.metastore.try.direct.sql.ddl=false"}); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } - @AfterClass public static void tearDown() throws Exception { LOG.info("Shutting down metastore."); @@ -90,7 +75,11 @@ public class TestHiveMetaStoreGetMetaConf { securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); - + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname, + MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); + metastoreConf.setBoolVar(HiveConf.ConfVars.METASTORE_TRY_DIRECT_SQL_DDL, false); + int msPort = MetaStoreUtils.startMetaStore(metastoreConf); hiveConf = new HiveConf(TestHiveMetaStoreGetMetaConf.class); hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + msPort); @@ -101,8 +90,6 @@ public class TestHiveMetaStoreGetMetaConf { System.setProperty(HiveConf.ConfVars.PREEXECHOOKS.varname, " "); System.setProperty(HiveConf.ConfVars.POSTEXECHOOKS.varname, " "); - - new Thread(new RunMS()).start(); } @Before http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java index 922a4bf..c1646c3 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestHiveMetaStorePartitionSpecs.java @@ -49,7 +49,7 @@ import java.util.Map; public class TestHiveMetaStorePartitionSpecs { private static final Logger LOG = LoggerFactory.getLogger(TestHiveMetaStorePartitionSpecs.class); - private static final String msPort = "20102"; + private static int msPort; private static HiveConf hiveConf; private static SecurityManager securityManager; @@ -73,18 +73,6 @@ public class TestHiveMetaStorePartitionSpecs { } } - private static class RunMS implements Runnable { - - @Override - public void run() { - try { - HiveMetaStore.main(new String[]{"-v", "-p", msPort, "--hiveconf", - "hive.metastore.expression.proxy=" + MockPartitionExpressionForMetastore.class.getCanonicalName()}); - } catch (Throwable t) { - LOG.error("Exiting. Got exception from metastore: ", t); - } - } - } @AfterClass public static void tearDown() throws Exception { @@ -95,10 +83,10 @@ public class TestHiveMetaStorePartitionSpecs { @BeforeClass public static void startMetaStoreServer() throws Exception { - Thread t = new Thread(new RunMS()); - t.start(); - Thread.sleep(10000); - + HiveConf metastoreConf = new HiveConf(); + metastoreConf.setClass(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS.varname, + MockPartitionExpressionForMetastore.class, PartitionExpressionProxy.class); + msPort = MetaStoreUtils.startMetaStore(metastoreConf); securityManager = System.getSecurityManager(); System.setSecurityManager(new NoExitSecurityManager()); hiveConf = new HiveConf(TestHiveMetaStorePartitionSpecs.class); http://git-wip-us.apache.org/repos/asf/hive/blob/0ea9d35d/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java index ee95e16..9dfcd54 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveRemote.java @@ -22,7 +22,7 @@ import java.io.IOException; import java.net.ServerSocket; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.HiveMetaStore; +import org.apache.hadoop.hive.metastore.MetaStoreUtils; import org.apache.hadoop.util.StringUtils; /** @@ -34,43 +34,13 @@ import org.apache.hadoop.util.StringUtils; public class TestHiveRemote extends TestHive { /** - * Starts a remote metastore - */ - private static class RunMS implements Runnable { - String port; - - public RunMS(String port) { - this.port = port; - } - - @Override - public void run() { - try { - HiveMetaStore.main(new String[] { port }); - } catch (Throwable e) { - e.printStackTrace(System.err); - assert false; - } - } - - } - - /** * Start a remote metastore and initialize a Hive object pointing at it. */ @Override protected void setUp() throws Exception { super.setUp(); hiveConf = new HiveConf(this.getClass()); - String port = findFreePort(); - hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + port); - - Thread t = new Thread(new RunMS(port)); - t.start(); - - // Wait a little bit for the metastore to start. - Thread.sleep(5000); - + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:" + MetaStoreUtils.startMetaStore()); try { hm = Hive.get(hiveConf);