OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/00275b71 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/00275b71 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/00275b71 Branch: refs/heads/oya Commit: 00275b71a626a5035b9802309aadf7df378c5e59 Parents: a16de89 Author: Robert Kanter <rkan...@cloudera.com> Authored: Tue Sep 20 15:16:29 2016 -0700 Committer: Robert Kanter <rkan...@cloudera.com> Committed: Tue Sep 20 15:16:29 2016 -0700 ---------------------------------------------------------------------- .../command/coord/TestCoordRerunXCommand.java | 2 +- ...TestHAPartitionDependencyManagerService.java | 1 + .../org/apache/oozie/test/XHCatTestCase.java | 1 + .../java/org/apache/oozie/test/XTestCase.java | 82 +++++++++++++++----- .../oozie/example/TestLocalOozieExample.java | 1 - .../org/apache/oozie/test/WorkflowTest.java | 1 - release-log.txt | 1 + .../action/hadoop/TestPigActionExecutor.java | 1 - .../apache/oozie/action/hadoop/TestPigMain.java | 1 - 9 files changed, 67 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java index cdfc9d6..891e6ec 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordRerunXCommand.java @@ -569,7 +569,7 @@ public class TestCoordRerunXCommand extends XDataTestCase { * @throws Exception */ public void testCoordRerunCleanupForHCat() throws Exception { - + super.setupHCatalogServer(); services = super.setupServicesForHCatalog(); services.init(); http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java b/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java index d681d42..3e1df07 100644 --- a/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java +++ b/core/src/test/java/org/apache/oozie/service/TestHAPartitionDependencyManagerService.java @@ -47,6 +47,7 @@ public class TestHAPartitionDependencyManagerService extends ZKXTestCase { protected void setUp() throws Exception { super.setUp(); services = super.setupServicesForHCatalog(Services.get()); + super.setupHCatalogServer(); // disable recovery service services.getConf().setInt(RecoveryService.CONF_SERVICE_INTERVAL, 1000000); // disable regular cache purge http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java b/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java index 85ee1f2..2adbee7 100644 --- a/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XHCatTestCase.java @@ -40,6 +40,7 @@ public abstract class XHCatTestCase extends XFsTestCase { @Override protected void setUp() throws Exception { super.setUp(); + super.setupHCatalogServer(); hcatServer = super.getHCatalogServer(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/core/src/test/java/org/apache/oozie/test/XTestCase.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/test/XTestCase.java b/core/src/test/java/org/apache/oozie/test/XTestCase.java index 3dd99d7..70f77be 100644 --- a/core/src/test/java/org/apache/oozie/test/XTestCase.java +++ b/core/src/test/java/org/apache/oozie/test/XTestCase.java @@ -130,12 +130,12 @@ public abstract class XTestCase extends TestCase { OOZIE_SRC_DIR = new File(OOZIE_SRC_DIR, "core"); } if (!OOZIE_SRC_DIR.exists()) { - System.err.println(); - System.err.println("Could not determine project root directory"); - System.err.println(); - System.exit(-1); + // We're probably being run from outside of Oozie (e.g. MiniOozie), so just use a dummy location here. + // Anything that uses this location should have a fallback anyway. + OOZIE_SRC_DIR = new File("."); + } else { + OOZIE_SRC_DIR = OOZIE_SRC_DIR.getParentFile(); } - OOZIE_SRC_DIR = OOZIE_SRC_DIR.getParentFile(); String testPropsFile = System.getProperty(OOZIE_TEST_PROPERTIES, "test.properties"); File file = new File(testPropsFile).isAbsolute() @@ -343,7 +343,7 @@ public abstract class XTestCase extends TestCase { else { // If we still can't find it, then exit System.err.println(); - System.err.println(XLog.format("Custom configuration file for testing does no exist [{0}]", + System.err.println(XLog.format("Custom configuration file for testing does not exist [{0}]", source.getAbsolutePath())); System.err.println(); System.exit(-1); @@ -369,8 +369,26 @@ public abstract class XTestCase extends TestCase { File actionConfDir = new File(testCaseConfDir, "action-conf"); actionConfDir.mkdir(); source = new File(OOZIE_SRC_DIR, "core/src/test/resources/hadoop-config.xml"); + InputStream hadoopConfigResourceStream = null; + if (!source.exists()) { + // If we can't find it, try using the class loader (useful if we're using XTestCase from outside core) + URL sourceURL = getClass().getClassLoader().getResource("hadoop-config.xml"); + if (sourceURL != null) { + hadoopConfigResourceStream = sourceURL.openStream(); + } + else { + // If we still can't find it, then exit + System.err.println(); + System.err.println(XLog.format("hadoop-config.xml configuration file for testing does not exist [{0}]", + source.getAbsolutePath())); + System.err.println(); + System.exit(-1); + } + } else { + hadoopConfigResourceStream = new FileInputStream(source); + } target = new File(hadoopConfDir, "hadoop-site.xml"); - IOUtils.copyStream(new FileInputStream(source), new FileOutputStream(target)); + IOUtils.copyStream(hadoopConfigResourceStream, new FileOutputStream(target)); if (System.getProperty("oozielocal.log") == null) { setSystemProperty("oozielocal.log", "/tmp/oozielocal.log"); @@ -400,10 +418,14 @@ public abstract class XTestCase extends TestCase { os.close(); } - if (System.getProperty("oozie.test.metastore.server", "true").equals("true")) { + if (System.getProperty("oozie.test.metastore.server", "false").equals("true")) { setupHCatalogServer(); } + if (System.getProperty("oozie.test.hive.server.2", "false").equals("true")) { + setupHiveServer2(); + } + // Cleanup any leftover database data to make sure we start each test with an empty database if (cleanUpDBTables) { cleanUpDBTables(); @@ -415,10 +437,8 @@ public abstract class XTestCase extends TestCase { */ @Override protected void tearDown() throws Exception { - if (hiveserver2 != null && hiveserver2.isStarted()) { - hiveserver2.stop(); - hiveserver2 = null; - } + tearDownHiveServer2(); + tearDownHCatalogServer(); resetSystemProperties(); sysProps = null; testCaseDir = null; @@ -881,6 +901,7 @@ public abstract class XTestCase extends TestCase { private static MiniMRCluster mrCluster = null; private static MiniHCatServer hcatServer = null; private static MiniHS2 hiveserver2 = null; + private static HiveConf hs2Config = null; private void setUpEmbeddedHadoop(String testCaseDir) throws Exception { if (dfsCluster == null && mrCluster == null) { @@ -989,7 +1010,7 @@ public abstract class XTestCase extends TestCase { return conf; } - private void setupHCatalogServer() throws Exception { + protected void setupHCatalogServer() throws Exception { if (hcatServer == null) { hcatServer = new MiniHCatServer(RUNMODE.SERVER, createJobConf()); hcatServer.start(); @@ -997,17 +1018,38 @@ public abstract class XTestCase extends TestCase { } } + protected void tearDownHCatalogServer() throws Exception { + // TODO: This doesn't properly shutdown the metastore. For now, just keep the current one running once it's been started +// if (hcatServer != null) { +// hcatServer.shutdown(); +// hcatServer = null; +// log.info("Metastore server shutdown"); +// } + } + protected void setupHiveServer2() throws Exception { if (hiveserver2 == null) { setSystemProperty("test.tmp.dir", getTestCaseDir()); - // Make HS2 use our Mini cluster by copying all configs to HiveConf; also had to hack MiniHS2 - HiveConf hconf = new HiveConf(); - Configuration jobConf = createJobConf(); - for (Map.Entry<String, String> pair: jobConf) { - hconf.set(pair.getKey(), pair.getValue()); + // We cache the HS2 config because it's expensive to build + if (hs2Config == null) { + // Make HS2 use our Mini cluster by copying all configs to HiveConf; also had to hack MiniHS2 + hs2Config = new HiveConf(); + Configuration jobConf = createJobConf(); + for (Map.Entry<String, String> pair : jobConf) { + hs2Config.set(pair.getKey(), pair.getValue()); + } } - hiveserver2 = new MiniHS2(hconf, dfsCluster.getFileSystem()); + hiveserver2 = new MiniHS2(hs2Config, dfsCluster.getFileSystem()); hiveserver2.start(new HashMap<String, String>()); + log.info("Hive Server 2 started at " + hiveserver2.getJdbcURL()); + } + } + + protected void tearDownHiveServer2() { + if (hiveserver2 != null && hiveserver2.isStarted()) { + hiveserver2.stop(); + hiveserver2 = null; + log.info("Hive Server 2 shutdown"); } } @@ -1036,6 +1078,8 @@ public abstract class XTestCase extends TestCase { catch (Exception ex) { System.out.println(ex); } + // This is tied to the MiniCluster because it inherits configs from there + hs2Config = null; } private static void shutdownMiniCluster2() { http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java ---------------------------------------------------------------------- diff --git a/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java b/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java index 2c819b7..9b7bfe4 100644 --- a/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java +++ b/examples/src/test/java/org/apache/oozie/example/TestLocalOozieExample.java @@ -34,7 +34,6 @@ public class TestLocalOozieExample extends XFsTestCase { @Override protected void setUp() throws Exception { - System.setProperty("oozie.test.metastore.server", "false"); super.setUp(); oozieLocalLog = System.getProperty("oozielocal.log"); System.setProperty("oozielocal.log", getTestCaseDir()+"/oozielocal.log"); http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java ---------------------------------------------------------------------- diff --git a/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java b/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java index 64027d9..2845f0a 100644 --- a/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java +++ b/minitest/src/test/java/org/apache/oozie/test/WorkflowTest.java @@ -42,7 +42,6 @@ public class WorkflowTest extends MiniOozieTestCase { @Override protected void setUp() throws Exception { - System.setProperty("oozie.test.metastore.server", "false"); System.setProperty(XLogService.LOG4J_FILE, "oozie-log4j.properties"); super.setUp(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 0dd42d0..71d9dfe 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.3.0 release (trunk - unreleased) +OOZIE-2273 MiniOozie does not work outside of Oozie (rkanter) OOZIE-2667 Optimize queries for DB export (gezapeti via rkanter) OOZIE-1793 Improve find bugs reporting for Oozie (rkanter) OOZIE-2572 SLA DURATION miss not shown when job is running for longer than expected time (satishsaley via puru) http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java index 09d3da3..25092ce 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigActionExecutor.java @@ -76,7 +76,6 @@ public class TestPigActionExecutor extends ActionExecutorTestCase { @Override protected void setUp() throws Exception { - System.setProperty("oozie.test.metastore.server", "false"); super.setUp(); PigTestCase.resetPigStats(); } http://git-wip-us.apache.org/repos/asf/oozie/blob/00275b71/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java index 1ea612d..081d319 100644 --- a/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java +++ b/sharelib/pig/src/test/java/org/apache/oozie/action/hadoop/TestPigMain.java @@ -40,7 +40,6 @@ public class TestPigMain extends PigTestCase { @Override protected void setUp() throws Exception { - System.setProperty("oozie.test.metastore.server", "false"); super.setUp(); SECURITY_MANAGER = System.getSecurityManager(); resetPigStats();
