http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java new file mode 100644 index 0000000..b419a47 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java @@ -0,0 +1,288 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.command.coord; + +import java.io.File; +import java.io.FileWriter; +import java.io.Reader; +import java.io.Writer; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.CoordinatorActionBean; +import org.apache.oozie.client.OozieClient; +import org.apache.oozie.coord.input.logic.TestCoordInputLogicPush; +import org.apache.oozie.coord.input.logic.TestCoordInputLogicPush.TEST_TYPE; +import org.apache.oozie.dependency.ActionDependency; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XHCatTestCase; +import org.apache.oozie.util.IOUtils; +import org.apache.oozie.util.Pair; +import org.apache.oozie.util.XConfiguration; + +public class TestCoordActionMissingDependenciesXCommand extends XHCatTestCase { + private Services services; + final String TABLE = "table1"; + final String DB_A = "db_a"; + final String DB_B = "db_b"; + final String DB_C = "db_c"; + final String DB_D = "db_d"; + final String DB_E = "db_e"; + final String DB_F = "db_f"; + + @Override + protected void setUp() throws Exception { + super.setUp(); + services = super.setupServicesForHCatalog(); + services.init(); + + } + + @Override + protected void tearDown() throws Exception { + services.destroy(); + super.tearDown(); + } + + public void testCoordActionPullDependencyMissing() throws Exception { + + Configuration conf = new XConfiguration(); + File appPathFile = new File(getTestCaseDir(), "coordinator.xml"); + + // CASE 1: Failure case i.e. multiple data-in instances + Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1); + Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml")); + IOUtils.copyCharStream(reader, writer); + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + conf.set("data_set_a", "file://" + getTestCaseDir() + "/input-data/a/${YEAR}/${DAY}"); + conf.set("data_set_b", "file://" + getTestCaseDir() + "/input-data/b/${YEAR}/${DAY}"); + conf.set("data_set_c", "file://" + getTestCaseDir() + "/input-data/c/${YEAR}/${DAY}"); + conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d/${YEAR}/${DAY}"); + conf.set("data_set_e", "file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}"); + + CoordSubmitXCommand sc = new CoordSubmitXCommand(conf); + String jobId = sc.call(); + new CoordMaterializeTransitionXCommand(jobId, 3600).call(); + List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand( + jobId + "@1").call(); + + Map<String, ActionDependency> dependencyMap = data.get(0).getSecond(); + assertEquals(6, dependencyMap.size()); + assertEquals(1, dependencyMap.get("A").getMissingDependencies().size()); + assertEquals(6, dependencyMap.get("B").getMissingDependencies().size()); + assertEquals(1, dependencyMap.get("C").getMissingDependencies().size()); + assertEquals(1, dependencyMap.get("D").getMissingDependencies().size()); + assertEquals(6, dependencyMap.get("E").getMissingDependencies().size()); + createTestCaseSubDir("input-data/a/2009/01/_SUCCESS".split("/")); + createTestCaseSubDir("input-data/b/2009/01/_SUCCESS".split("/")); + new CoordActionInputCheckXCommand(jobId + "@1", jobId).call(); + data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call(); + dependencyMap = data.get(0).getSecond(); + assertEquals(5, dependencyMap.size()); + assertNull(dependencyMap.get("A")); + assertEquals(5, dependencyMap.get("B").getMissingDependencies().size()); + assertTrue(dependencyMap.get("B").getMissingDependencies() + .contains("file://" + getTestCaseDir() + "/input-data/b/2009/31/_SUCCESS")); + assertTrue(dependencyMap.get("B").getMissingDependencies() + .contains("file://" + getTestCaseDir() + "/input-data/b/2009/30/_SUCCESS")); + assertTrue(dependencyMap.get("B").getMissingDependencies() + .contains("file://" + getTestCaseDir() + "/input-data/b/2009/29/_SUCCESS")); + assertTrue(dependencyMap.get("B").getMissingDependencies() + .contains("file://" + getTestCaseDir() + "/input-data/b/2009/28/_SUCCESS")); + assertTrue(dependencyMap.get("B").getMissingDependencies() + .contains("file://" + getTestCaseDir() + "/input-data/b/2009/27/_SUCCESS")); + assertEquals(1, dependencyMap.get("C").getMissingDependencies().size()); + assertEquals(1, dependencyMap.get("D").getMissingDependencies().size()); + assertEquals(dependencyMap.get("D").getMissingDependencies().get(0), + "file://" + getTestCaseDir() + "/input-data/d/2009/01"); + + assertEquals(6, dependencyMap.get("E").getMissingDependencies().size()); + + assertEquals(2, dependencyMap.get("F").getMissingDependencies().size()); + assertEquals(dependencyMap.get("F").getMissingDependencies().get(0), + "file://" + getTestCaseDir() + "/input-data/e/2009/01/_SUCCESS"); + assertEquals(dependencyMap.get("F").getMissingDependencies().get(1), "${coord:latest(0)}"); + } + + public void testCoordActionPushDependencyMissing() throws Exception { + createTestTables(); + Configuration conf = new XConfiguration(); + File appPathFile = new File(getTestCaseDir(), "coordinator.xml"); + Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1); + Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml")); + IOUtils.copyCharStream(reader, writer); + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + String datasetPrefix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa"; + String datasetSuffix = "hcat://" + getMetastoreAuthority() + "/"; + + conf.set("data_set_a", datasetSuffix.toString() + DB_A + datasetPrefix); + conf.set("data_set_b", datasetSuffix.toString() + DB_B + datasetPrefix); + conf.set("data_set_c", datasetSuffix.toString() + DB_C + datasetPrefix); + conf.set("data_set_d", datasetSuffix.toString() + DB_D + datasetPrefix); + conf.set("data_set_e", datasetSuffix.toString() + DB_E + datasetPrefix); + + CoordSubmitXCommand sc = new CoordSubmitXCommand(conf); + String jobId = sc.call(); + new CoordMaterializeTransitionXCommand(jobId, 3600).call(); + List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand( + jobId + "@1").call(); + + assertEquals("${coord:latestRange(-5,0)}", CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst())); + + Map<String, ActionDependency> dependencyMap = data.get(0).getSecond(); + assertEquals(6, dependencyMap.size()); + assertEquals(1, dependencyMap.get("A").getMissingDependencies().size()); + assertEquals(6, dependencyMap.get("B").getMissingDependencies().size()); + assertEquals(1, dependencyMap.get("C").getMissingDependencies().size()); + assertEquals(1, dependencyMap.get("D").getMissingDependencies().size()); + assertEquals(6, dependencyMap.get("E").getMissingDependencies().size()); + + addPartition(DB_A, TABLE, "dt=200901;country=usa"); + new CoordPushDependencyCheckXCommand(jobId + "@1").call(); + data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call(); + dependencyMap = data.get(0).getSecond(); + assertEquals(5, dependencyMap.size()); + assertNull(dependencyMap.get("A")); + + addPartition(DB_B, TABLE, "dt=200901;country=usa"); + addPartition(DB_B, TABLE, "dt=200931;country=usa"); + addPartition(DB_B, TABLE, "dt=200930;country=usa"); + addPartition(DB_B, TABLE, "dt=200929;country=usa"); + addPartition(DB_B, TABLE, "dt=200928;country=usa"); + addPartition(DB_B, TABLE, "dt=200927;country=usa"); + + new CoordPushDependencyCheckXCommand(jobId + "@1").call(); + data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call(); + dependencyMap = data.get(0).getSecond(); + assertEquals(4, dependencyMap.size()); + assertNull(dependencyMap.get("B")); + } + + public void testCoordActionPullPushDependencyMissing() throws Exception { + createTestTables(); + Configuration conf = new XConfiguration(); + File appPathFile = new File(getTestCaseDir(), "coordinator.xml"); + Reader reader = IOUtils.getResourceAsReader("coord-multiple-output-instance5.xml", -1); + Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml")); + IOUtils.copyCharStream(reader, writer); + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + String datasetPrefix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa"; + String datasetSuffix = "hcat://" + getMetastoreAuthority() + "/"; + + conf.set("data_set_a", datasetSuffix.toString() + DB_A + datasetPrefix); + conf.set("data_set_b", datasetSuffix.toString() + DB_B + datasetPrefix); + conf.set("data_set_c", datasetSuffix.toString() + DB_C + datasetPrefix); + conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d/${YEAR}/${DAY}"); + conf.set("data_set_e", "file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}"); + + CoordSubmitXCommand sc = new CoordSubmitXCommand(conf); + String jobId = sc.call(); + new CoordMaterializeTransitionXCommand(jobId, 3600).call(); + List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand( + jobId + "@1").call(); + + assertEquals("file://" + getTestCaseDir() + "/input-data/d/2009/01", + CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst())); + + Map<String, ActionDependency> dependencyMap = data.get(0).getSecond(); + assertEquals(6, dependencyMap.size()); + assertEquals(1, dependencyMap.get("A").getMissingDependencies().size()); + assertEquals(6, dependencyMap.get("B").getMissingDependencies().size()); + assertEquals(1, dependencyMap.get("C").getMissingDependencies().size()); + assertEquals(1, dependencyMap.get("D").getMissingDependencies().size()); + assertEquals(6, dependencyMap.get("E").getMissingDependencies().size()); + + addPartition(DB_A, TABLE, "dt=200901;country=usa"); + addPartition(DB_B, TABLE, "dt=200901;country=usa"); + addPartition(DB_B, TABLE, "dt=200931;country=usa"); + addPartition(DB_B, TABLE, "dt=200930;country=usa"); + addPartition(DB_B, TABLE, "dt=200929;country=usa"); + addPartition(DB_B, TABLE, "dt=200928;country=usa"); + addPartition(DB_B, TABLE, "dt=200927;country=usa"); + createTestCaseSubDir("input-data/d/2009/01".split("/")); + + new CoordActionInputCheckXCommand(jobId + "@1", jobId).call(); + new CoordPushDependencyCheckXCommand(jobId + "@1").call(); + data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call(); + dependencyMap = data.get(0).getSecond(); + assertEquals(3, dependencyMap.size()); + assertNull(dependencyMap.get("B")); + assertNull(dependencyMap.get("D")); + } + + public void testCoordActionInputLogicMissing() throws Exception { + createTestTables(); + + Configuration conf = TestCoordInputLogicPush.getConfForCombine("file://" + getTestCaseDir(), + "hcat://" + getMetastoreAuthority()); + conf.set("initial_instance_a", "2014-10-07T00:00Z"); + conf.set("initial_instance_b", "2014-10-07T00:00Z"); + + String inputLogic = + // @formatter:off + "<and name=\"test\">" + "<data-in dataset=\"A\" />" + "<data-in dataset=\"B\" />" + "</and>"; + // @formatter:on + String jobId = TestCoordInputLogicPush.submitCoord(getTestCaseDir(), "coord-inputlogic-combine.xml", conf, + inputLogic, TEST_TYPE.CURRENT_SINGLE, TEST_TYPE.CURRENT_SINGLE, TEST_TYPE.CURRENT_RANGE, + TEST_TYPE.LATEST_RANGE); + + new CoordMaterializeTransitionXCommand(jobId, 3600).call(); + List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand( + jobId + "@1").call(); + Map<String, ActionDependency> dependencyMap = data.get(0).getSecond(); + assertEquals(6, dependencyMap.size()); + + assertNull(CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst())); + + createTestCaseSubDir("input-data/b/2014/10/08/_SUCCESS".split("/")); + addPartition(DB_A, TABLE, "dt=20141008;country=usa"); + + new CoordActionInputCheckXCommand(jobId + "@1", jobId).call(); + new CoordPushDependencyCheckXCommand(jobId + "@1").call(); + new CoordActionInputCheckXCommand(jobId + "@1", jobId).call(); + + data = new CoordActionMissingDependenciesXCommand(jobId + "@1").call(); + dependencyMap = data.get(0).getSecond(); + assertEquals(4, dependencyMap.size()); + assertNull(dependencyMap.get("A")); + assertNull(dependencyMap.get("B")); + + } + + private void createSingleTestDB(String db) throws Exception { + dropTable(db, TABLE, true); + dropDatabase(db, true); + createDatabase(db); + createTable(db, TABLE, "dt,country"); + } + + private void createTestTables() throws Exception { + createSingleTestDB(DB_A); + createSingleTestDB(DB_B); + createSingleTestDB(DB_C); + createSingleTestDB(DB_D); + createSingleTestDB(DB_E); + createSingleTestDB(DB_F); + + } + +}
http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java b/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java index 6684a1f..d66aace 100644 --- a/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java +++ b/core/src/test/java/org/apache/oozie/coord/input/logic/TestCoordInputLogicPush.java @@ -64,9 +64,10 @@ public class TestCoordInputLogicPush extends XHCatTestCase { private String server; private static final String table = "table1"; - final long TIME_DAYS = 60 * 60 * 1000 * 24; - enum TEST_TYPE { + public final static long TIME_DAYS = 60 * 60 * 1000 * 24; + + public static enum TEST_TYPE { CURRENT_SINGLE, CURRENT_RANGE, LATEST_SINGLE, LATEST_RANGE; }; @@ -132,7 +133,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { "</or>"; //@formatter:on conf.set("partitionName", "test"); - String jobId = _testCoordSubmit("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE); + String jobId = submitCoord("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE); String input = addPartition("db_b", "table1", "dt=20141008;country=usa"); @@ -168,7 +169,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { "</and>"; //@formatter:on conf.set("partitionName", "test"); - final String jobId = _testCoordSubmit("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE); + final String jobId = submitCoord("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE); String input1 = addPartition("db_a", "table1", "dt=20141008;country=usa"); String input2 = addPartition("db_b", "table1", "dt=20141008;country=usa"); @@ -215,7 +216,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { "</and>"; //@formatter:on conf.set("partitionName", "test"); - final String jobId = _testCoordSubmit("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_RANGE, + final String jobId = submitCoord("coord-inputlogic-hcat.xml", conf, inputLogic, TEST_TYPE.CURRENT_RANGE, TEST_TYPE.LATEST_RANGE); List<String> inputPartition = createPartitionWithTime("db_a", now, 0, 1, 2); inputPartition.addAll(createPartitionWithTime("db_c", now, 0, 1, 2)); @@ -246,7 +247,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { "<data-in dataset=\"B\" />" + "</and>"; //@formatter:on - String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE); + String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE); List<String> inputDir = createDirWithTime("input-data/b/", now, 0, 1, 2, 3, 4, 5); inputDir.addAll(createPartitionWithTime("db_a", now, 0, 1, 2, 3, 4, 5)); @@ -280,7 +281,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { "<data-in dataset=\"B\"/>" + "</and>"; //@formatter:on - String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE, + String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE, TEST_TYPE.CURRENT_RANGE); List<String> inputDir = createDirWithTime("input-data/b/", now, 0, 1, 2, 3, 4, 5); @@ -322,7 +323,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { "</or>"; //@formatter:on - String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE); + String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE); List<String> inputDir = createDirWithTime("input-data/b/", now, 0, 1, 2, 3, 4, 5); inputDir.addAll(createPartitionWithTime("db_a", now, 0, 1, 2, 3, 4, 5)); @@ -351,7 +352,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { "<data-in name=\"testB\" dataset=\"B\" />" + "</and>"; //@formatter:on - String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE); + String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE); String input1 = createTestCaseSubDir("input-data/b/2014/10/08/_SUCCESS".split("/")); String input2 = addPartition("db_a", "table1", "dt=20141008;country=usa"); @@ -394,7 +395,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { "</and>"; //@formatter:on - String jobId = _testCoordSubmit("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE); + String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE); String input1 = createTestCaseSubDir(("input-data/d/" + sd.format(now) + "/_SUCCESS").split("/")); sd = new SimpleDateFormat("yyyyMMdd"); @@ -429,31 +430,31 @@ public class TestCoordInputLogicPush extends XHCatTestCase { conf.set("db_e", "db_e"); conf.set("db_f", "db_f"); conf.set("table", table); - conf.set("wfPath", getWFPath()); + conf.set("wfPath", getWFPath(getTestCaseFileUri("workflow.xml"))); conf.set("partitionName", "test"); return conf; + } + public Configuration getConfForCombine() throws Exception { + return getConfForCombine("file://" + getTestCaseDir(), "hcat://" + getMetastoreAuthority()); } - private Configuration getConfForCombine() throws Exception { + public static Configuration getConfForCombine(String testCaseDir, String hcatURL) throws Exception { Configuration conf = new XConfiguration(); conf.set("start_time", "2014-10-08T00:00Z"); conf.set("end_time", "2015-10-08T00:00Z"); conf.set("initial_instance", "2014-10-08T00:00Z"); - conf.set("data_set_b", "file://" + getTestCaseDir() + "/input-data/b"); - conf.set("data_set_d", "file://" + getTestCaseDir() + "/input-data/d"); - conf.set("data_set_f", "file://" + getTestCaseDir() + "/input-data/f"); + conf.set("data_set_b", testCaseDir + "/input-data/b"); + conf.set("data_set_d", testCaseDir + "/input-data/d"); + conf.set("data_set_f", testCaseDir + "/input-data/f"); conf.set("start_time", "2014-10-08T00:00Z"); conf.set("end_time", "2015-10-08T00:00Z"); conf.set("initial_instance_a", "2014-10-08T00:00Z"); conf.set("initial_instance_b", "2014-10-08T00:00Z"); - - String dataset1 = "hcat://" + getMetastoreAuthority(); - - conf.set("data_set", dataset1.toString()); + conf.set("data_set", hcatURL); conf.set("db_a", "db_a"); conf.set("db_b", "db_b"); conf.set("db_c", "db_c"); @@ -461,16 +462,19 @@ public class TestCoordInputLogicPush extends XHCatTestCase { conf.set("db_e", "db_e"); conf.set("db_f", "db_f"); conf.set("table", table); - conf.set("wfPath", getWFPath()); + conf.set("wfPath", getWFPath(testCaseDir + "/workflow.xml")); conf.set("partitionName", "test"); - return conf; - } - private String _testCoordSubmit(String coordinatorXml, Configuration conf, String inputLogic, TEST_TYPE... testType) + public String submitCoord(String coordinatorXml, Configuration conf, String inputLogic, TEST_TYPE... testType) throws Exception { - String appPath = "file://" + getTestCaseDir() + File.separator + "coordinator.xml"; + return submitCoord(getTestCaseDir(), coordinatorXml, conf, inputLogic, testType); + } + + public static String submitCoord(String testCaseDir, String coordinatorXml, Configuration conf, String inputLogic, + TEST_TYPE... testType) throws Exception { + String appPath = "file://" + testCaseDir + File.separator + "coordinator.xml"; String content = IOUtils.getResourceAsString(coordinatorXml, -1); content = content.replaceAll("=input-logic=", inputLogic); @@ -504,8 +508,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { return coordId; } - public String getWFPath() throws Exception { - String workflowUri = getTestCaseFileUri("workflow.xml"); + public static String getWFPath(String workflowUri) throws Exception { String appXml = "<workflow-app xmlns='uri:oozie:workflow:0.1' name='map-reduce-wf'> " + "<start to='end' /> " + "<end name='end' /> " + "</workflow-app>"; @@ -513,7 +516,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { return workflowUri; } - private void writeToFile(String appXml, String appPath) throws IOException { + private static void writeToFile(String appXml, String appPath) throws IOException { File wf = new File(URI.create(appPath)); PrintWriter out = null; try { @@ -593,7 +596,7 @@ public class TestCoordInputLogicPush extends XHCatTestCase { return conf; } - private String getEnumText(TEST_TYPE testType) { + private static String getEnumText(TEST_TYPE testType) { switch (testType) { case LATEST_SINGLE: return "<instance>\\${coord:latest(0)}</instance>"; http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java b/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java index 0e74f2d..4fc8653 100644 --- a/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java +++ b/core/src/test/java/org/apache/oozie/servlet/MockCoordinatorEngineService.java @@ -39,8 +39,11 @@ import org.apache.oozie.client.CoordinatorAction; import org.apache.oozie.client.CoordinatorJob; import org.apache.oozie.client.CoordinatorJob.Execution; import org.apache.oozie.client.rest.RestConstants; +import org.apache.oozie.command.CommandException; +import org.apache.oozie.dependency.ActionDependency; import org.apache.oozie.service.CoordinatorEngineService; import org.apache.oozie.util.DateUtils; +import org.apache.oozie.util.Pair; public class MockCoordinatorEngineService extends CoordinatorEngineService { public static final String JOB_ID = "coord-job-C-"; @@ -250,6 +253,13 @@ public class MockCoordinatorEngineService extends CoordinatorEngineService { } @Override + public List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> getCoordActionMissingDependencies( + String id, String actions, String dates) throws CommandException { + did = RestConstants.COORD_ACTION_MISSING_DEPENDENCIES; + return new ArrayList<Pair<CoordinatorActionBean, Map<String, ActionDependency>>>() ; + } + + @Override public CoordinatorJobInfo suspendJobs(String filter, int start, int length) throws CoordinatorEngineException { did = RestConstants.JOBS; http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/core/src/test/resources/coord-multiple-output-instance5.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-multiple-output-instance5.xml b/core/src/test/resources/coord-multiple-output-instance5.xml new file mode 100644 index 0000000..5932860 --- /dev/null +++ b/core/src/test/resources/coord-multiple-output-instance5.xml @@ -0,0 +1,108 @@ +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + --> +<coordinator-app xmlns="uri:oozie:coordinator:0.5" name="NAME" + frequency="${coord:days(1)}" start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" + timezone="UTC"> + <controls> + <concurrency>2</concurrency> + <execution>LIFO</execution> + </controls> + <datasets> + + <dataset name="a" frequency="${coord:days(1)}" + initial-instance="2009-01-01T01:00Z" timezone="UTC"> + <uri-template>${data_set_a} + </uri-template> + </dataset> + + <dataset name="b" frequency="${coord:days(1)}" + initial-instance="2009-01-01T01:00Z" timezone="UTC"> + <uri-template>${data_set_b} + </uri-template> + </dataset> + <dataset name="c" frequency="${coord:days(1)}" + initial-instance="2009-01-01T01:00Z" timezone="UTC"> + <uri-template>${data_set_c} + </uri-template> + </dataset> + <dataset name="d" frequency="${coord:days(1)}" + initial-instance="2009-01-01T01:00Z" timezone="UTC"> + <uri-template>${data_set_d} + </uri-template> + <done-flag></done-flag> + </dataset> + + <dataset name="e" frequency="${coord:days(1)}" + initial-instance="2009-01-01T01:00Z" timezone="UTC"> + <uri-template>${data_set_e} + </uri-template> + </dataset> + + <dataset name="local_a" frequency="${coord:days(1)}" + initial-instance="2009-02-01T01:00Z" timezone="UTC"> + <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY} + </uri-template> + </dataset> + </datasets> + <input-events> + <data-in name="A" dataset="a"> + <instance>${coord:current(0)}</instance> + </data-in> + <data-in name="B" dataset="b"> + <start-instance>${coord:current(-5)}</start-instance> + <end-instance>${coord:current(0)}</end-instance> + </data-in> + <data-in name="C" dataset="c"> + <start-instance>${coord:latest(-5)}</start-instance> + <end-instance>${coord:latest(0)}</end-instance> + </data-in> + <data-in name="D" dataset="d"> + <instance>${coord:current(0)}</instance> + </data-in> + <data-in name="E" dataset="e"> + <start-instance>${coord:current(-5)}</start-instance> + <end-instance>${coord:current(0)}</end-instance> + </data-in> + <data-in name="F" dataset="e"> + <instance>${coord:current(0)}</instance> + <instance>${coord:latest(0)}</instance> + </data-in> + </input-events> + <output-events> + <data-out name="LOCAL_A" dataset="local_a"> + <instance>${coord:current(0)}</instance> + </data-out> + </output-events> + <action> + <workflow> + <app-path>hdfs:///tmp/workflows/</app-path> + <configuration> + <property> + <name>inputA</name> + <value>${coord:dataIn('A')}</value> + </property> + <property> + <name>inputB</name> + <value>${coord:dataOut('LOCAL_A')}</value> + </property> + </configuration> + </workflow> + </action> +</coordinator-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index 2dbbd4c..67838fc 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -999,6 +999,31 @@ All other arguments are optional: $oozie job -slaenable <bundle_job_id> [-action 1,3-4,7-40] [-date 2009-01-01T01:00Z::2009-05-31T23:59Z,2009-11-10T01:00Z::2009-12-31T22:00Z] [-coordinator <List_of_coord_names/ids] </verbatim> +---+++ Getting missing dependencies of coordinator action(s) + * Coordination action id can be specified directly for getting missing dependencies of a single action. + * To get information on multiple actions, either -action or -date option can be specified with the coordinator job id. + * missingdeps command doesn't recompute dependencies. It list missing dependencies which were last computed. + * Oozie checks missing dependencies sequentially, and it will stop on first missing dependency. =Blocked On= is the first missing dependency for action. So, there could be a chance that Oozie will report some missing dependencies, but it might be present. To resolve the waiting issue, one should fix the blockedOn missing dependency. + * For input logic, missingdeps command doesn't compute input-logic expression. It will report everything which is missing or not computed. +<verbatim> +oozie job -oozie http://localhost:11000/oozie -missingDependencies 0000000-170104141851590-oozie-puru-C -action 1 +oozie job -oozie http://localhost:11000/oozie -missingDependencies 0000000-170104141851590-oozie-puru-C@1 + +id : 1 +blockedOn : hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/00/_SUCCESS + +dataSet : input-1 +missingDependencies : + hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS + +dataSet : input-2 +missingDependencies : + hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/01/00/_SUCCESS + hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/40/_SUCCESS + hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/20/_SUCCESS + hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/00/_SUCCESS +$ +</verbatim> ---++ Jobs Operations ---+++ Checking the Status of multiple Workflow Jobs http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/docs/src/site/twiki/WebServicesAPI.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WebServicesAPI.twiki b/docs/src/site/twiki/WebServicesAPI.twiki index 8406da6..03f05d2 100644 --- a/docs/src/site/twiki/WebServicesAPI.twiki +++ b/docs/src/site/twiki/WebServicesAPI.twiki @@ -1616,6 +1616,55 @@ PUT /oozie/v2/job/0000003-140319184715726-oozie-puru-C?action=sla-enable&action- </verbatim> Will enable SLA alert for actions 1,14,17,18,19,20. +---+++ Getting missing dependencies of coordinator action(s) + +<verbatim> +GET oozie/v2/job/0000000-170104115137443-oozie-puru-C?show=missing-dependencies&action-list=1,20 +</verbatim> + +*Response* + +<verbatim> +HTTP/1.1 200 OK +Content-Type: application/json;charset=UTF-8 + +{ +"missingDependencies": +[{ + "blockedOn": "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/00/_SUCCESS", + "dataSets": + [ + { + "missingDependencies": + [ + "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/00/00/_SUCCESS" + ], + "dataSet": "input-2" + } + ], + "id": 1 + }, + { + "blockedOn": "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/20/00/_SUCCESS", + "dataSets": + [ + { + "missingDependencies": + [ + "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/20/00/_SUCCESS", + "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/19/40/_SUCCESS", + "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/19/20/_SUCCESS", + "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/19/00/_SUCCESS", + "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/18/40/_SUCCESS", + "hdfs://localhost:9000/user/purushah/examples/input-data/rawLogs/2010/01/01/18/20/_SUCCESS" + ], + "dataSet": "input-2" + } + ], + "id": 20 + }] +} +</verbatim> ---++++ Jobs Information A HTTP GET request retrieves workflow and coordinator jobs information. http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index cd40ea6..081ead8 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2770 Show missing dependencies for coord actions (puru) OOZIE-2630 Oozie Coordinator EL Functions to get first day of the week/month (satishsaley) OOZIE-2771 Allow retrieving keystore and truststore passwords from Hadoop Credential Provider (asasvari via abhishekbafna) OOZIE-2619 Make Hive action defaults to match hive defaults when running from command line (venkatnrangan via abhishekbafna) http://git-wip-us.apache.org/repos/asf/oozie/blob/9035d91c/webapp/src/main/webapp/oozie-console.js ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js index 7b20e91..38989c7 100644 --- a/webapp/src/main/webapp/oozie-console.js +++ b/webapp/src/main/webapp/oozie-console.js @@ -1484,10 +1484,9 @@ function coordJobDetailsPopup(response, request) { var actionStatus = thisGrid.store.data.items[rowIndex].data; var workflowId = actionStatus["externalId"]; if(workflowId == null) { - jobDetailsTab.getComponent('coord_job_log').show(); - Ext.getCmp('actions_text_box').setValue(actionStatus["id"].split("@")[1]); - Ext.getCmp('search_filter_box').setValue('recent=5m'); - fetchLogs(coordJobId, actionsTextBox.getValue()); + jobDetailsTab.getComponent('coord_action_missing_dependencies').show(); + Ext.getCmp('actions_missing_dependencies').setValue(actionStatus["id"].split("@")[1]); + viewMissingDependencies.execute(); } else { jobDetailsGridWindow(workflowId); @@ -1681,6 +1680,114 @@ function coordJobDetailsPopup(response, request) { }, frame: false }); + + var missingDependenciesTreeRoot = new Ext.tree.TreeNode({ + text: "Coord Action Missing Dependencies", + expanded: true + }); + + var missingDependenciesTabButton = new Ext.Button({ + text: 'Get Missing Dependencies', + ctCls: 'x-btn-over', + ctCls: 'spaces', + isFormField: true, + handler: function() { + viewMissingDependencies.execute(); + } + }); + + var missingDependenciesActionText = new Ext.form.Label({ + text : 'Enter Coordinator Action number : ', + ctCls: 'spaces' + }); + var missingDependenciesTextBox = new Ext.form.TextField({ + fieldLabel: 'missingDependenciesAction', + id: 'actions_missing_dependencies', + name: 'missingDependenciesAction', + width: 150, + value: '' + }); + + var viewMissingDependencies = new Ext.Action({ + text: " ", + icon: 'ext-2.2/resources/images/default/grid/refresh.gif', + handler: function() { + Ext.Ajax.request({ + url: getOozieBase() + 'job/' + coordJobId + '?show=missing-dependencies&action-list=' + + missingDependenciesTextBox.getValue(), + success: function(response, request) { + var jsonData = JSON.parse(response.responseText); + var missingDependencies = jsonData["missingDependencies"]; + while (missingDependenciesTreeRoot.hasChildNodes()) { + var child = missingDependenciesTreeRoot.firstChild; + missingDependenciesTreeRoot.removeChild(child); + } + var missingDependenciesTree = treeNodeForMissingDependencies(missingDependencies, null, null, + missingDependenciesTreeRoot); + missingDependenciesTreeRoot.expand(false, true); + }, + failure : function(response, request) { + Ext.Msg.minWidth = 360; + Ext.Msg.alert(response.getResponseHeader["oozie-error-message"]); + } + }); + } + }); + + function treeNodeForMissingDependencies(json, rootText, blockedOn, rootNode) { + var result; + if( rootNode ){ + result = rootNode; + } else { + result = new Ext.tree.TreeNode({ + text: rootText + }); + } + if (typeof json === 'object') { + for (var i in json) { + if (json[i]) { + if (typeof json[i] == 'object') { + var c; + if (json[i]['id']) { + c = treeNodeForMissingDependencies(json[i]['dataSets'], json[i]['id'], json[i]['blockedOn']); + } + if (json[i]['dataSet']) { + c = treeNodeForMissingDependencies(json[i]['missingDependencies'], json[i]['dataSet'], blockedOn); + if( blockedOn ) { + var blockedOnChild = new Ext.tree.TreeNode({ + text: "Blocked On" + }); + blockedOnChild.appendChild(new Ext.tree.TreeNode({ + text: blockedOn + })); + result.appendChild(blockedOnChild); + blockedOn = null; + } + } + if (c) { + result.appendChild(c); + } + } + else if (typeof json[i] != 'function') { + result.appendChild(new Ext.tree.TreeNode({ + text: json[i] + })); + } + } + } + } + return result; + } + + var missingDependenciesArea = new Ext.tree.TreePanel({ + autoScroll: true, + useArrows: true, + height: 430, + deferredRender: false, + root: missingDependenciesTreeRoot, + tbar: [missingDependenciesActionText, missingDependenciesTextBox, missingDependenciesTabButton], + }); + function populateReruns(coordActionId) { var actionNum = rerunActionTextBox.getValue(); store.baseParams.scope = actionNum; @@ -1698,6 +1805,7 @@ function coordJobDetailsPopup(response, request) { var jobDetailsTab = new Ext.TabPanel({ activeTab: 0, autoHeight: true, + layoutOnTabChange: true, deferredRender: false, items: [ { title: 'Coord Job Info', @@ -1751,6 +1859,11 @@ function coordJobDetailsPopup(response, request) { tbar: [ rerunActionText, rerunActionTextBox, getRerunsButton] }, + { + title: 'Action Missing Dependencies', + id: 'coord_action_missing_dependencies', + items: missingDependenciesArea + } ]}); jobDetailsTab.addListener("tabchange", function(panel, selectedTab) { @@ -3169,8 +3282,8 @@ function initConsole() { viewConfig.execute(); serverVersion.execute(); if (isInstrumentationServiceEnabled == "true") { - viewInstrumentation.execute(); - } + viewInstrumentation.execute(); + } if (isMetricsInstrumentationServiceEnabled == "true") { viewMetrics.execute(); }
