Repository: oozie Updated Branches: refs/heads/master 50b92932f -> 18694b616
OOZIE-2987 Coord action missing dependencies should show URI template with unresolved dependencies Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/18694b61 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/18694b61 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/18694b61 Branch: refs/heads/master Commit: 18694b616f419ed958230cedb2dc0431a9f9cb5d Parents: 50b9293 Author: puru <purus...@yahoo-inc.com> Authored: Mon Jul 17 17:21:32 2017 -0700 Committer: puru <purus...@yahoo-inc.com> Committed: Mon Jul 17 17:21:32 2017 -0700 ---------------------------------------------------------------------- .../oozie/command/coord/CoordCommandUtils.java | 2 +- .../dependency/CoordOldInputDependency.java | 20 ++++++++++++++++-- .../oozie/dependency/ActionDependency.java | 9 ++++++++ ...tCoordActionMissingDependenciesXCommand.java | 22 ++++++++++++-------- release-log.txt | 1 + 5 files changed, 42 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java index f7f947f..fd1230b 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordCommandUtils.java @@ -928,7 +928,7 @@ public class CoordCommandUtils { CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies(); CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies(); String firstMissingDependencies = coordPullInputDependency.getFirstMissingDependency(); - if (StringUtils.isEmpty(firstMissingDependencies)) { + if (StringUtils.isEmpty(firstMissingDependencies) || firstMissingDependencies.trim().startsWith("${coord:")) { firstMissingDependencies = coordPushInputDependency.getFirstMissingDependency(); } return firstMissingDependencies; http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java b/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java index b0b6475..56aef1c 100644 --- a/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java +++ b/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java @@ -344,9 +344,20 @@ public class CoordOldInputDependency implements CoordInputDependency { Map.Entry<String, ActionDependency> entry = it.next(); ActionDependency dependency = entry.getValue(); dependency.getMissingDependencies().retainAll(missingSets); + + if (dependency.getUriTemplate() != null) { + for (int i = 0; i < dependency.getMissingDependencies().size(); i++) { + if (dependency.getMissingDependencies().get(i).trim().startsWith("${coord:")) { + dependency.getMissingDependencies().set(i, + dependency.getMissingDependencies().get(i) + " -> " + dependency.getUriTemplate()); + } + } + } + if (dependenciesMap.get(entry.getKey()).getMissingDependencies().isEmpty()) { it.remove(); } + } return dependenciesMap; } @@ -376,8 +387,10 @@ public class CoordOldInputDependency implements CoordInputDependency { } } if (event.getChildTextTrim(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, event.getNamespace()) != null) { + ActionDependency unResolvedDependency = getUnResolvedDependency(coordAction, event); dependency.getMissingDependencies() - .addAll(getUnResolvedDependency(coordAction, event).getMissingDependencies()); + .addAll(unResolvedDependency.getMissingDependencies()); + dependency.setUriTemplate(unResolvedDependency.getUriTemplate()); } dependenciesMap.put(event.getAttributeValue("name"), dependency); } @@ -387,10 +400,13 @@ public class CoordOldInputDependency implements CoordInputDependency { private ActionDependency getUnResolvedDependency(CoordinatorActionBean coordAction, Element event) throws JDOMException, URIHandlerException { String tmpUnresolved = event.getChildTextTrim(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, event.getNamespace()); - ActionDependency dependency = new ActionDependency(); StringBuilder nonResolvedList = new StringBuilder(); CoordCommandUtils.getResolvedList(getMissingDependencies(), new StringBuilder(), nonResolvedList); + ActionDependency dependency = new ActionDependency(); + if (nonResolvedList.length() > 0) { + Element eDataset = event.getChild("dataset", event.getNamespace()); + dependency.setUriTemplate(eDataset.getChild("uri-template", eDataset.getNamespace()).getTextTrim()); dependency.getMissingDependencies().add(tmpUnresolved); } return dependency; http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java b/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java index cc99038..a2e03c5 100644 --- a/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java +++ b/core/src/main/java/org/apache/oozie/dependency/ActionDependency.java @@ -25,6 +25,7 @@ public class ActionDependency { private List<String> missingDependencies; private List<String> availableDependencies; + private String uriTemplate; public ActionDependency() { this(new ArrayList<String>(), new ArrayList<String>()); @@ -43,4 +44,12 @@ public class ActionDependency { return availableDependencies; } + public String getUriTemplate() { + return uriTemplate; + } + + public void setUriTemplate(String uriTemplate) { + this.uriTemplate = uriTemplate; + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java index b419a47..b5ad9cd 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordActionMissingDependenciesXCommand.java @@ -118,7 +118,8 @@ public class TestCoordActionMissingDependenciesXCommand extends XHCatTestCase { assertEquals(2, dependencyMap.get("F").getMissingDependencies().size()); assertEquals(dependencyMap.get("F").getMissingDependencies().get(0), "file://" + getTestCaseDir() + "/input-data/e/2009/01/_SUCCESS"); - assertEquals(dependencyMap.get("F").getMissingDependencies().get(1), "${coord:latest(0)}"); + assertEquals(dependencyMap.get("F").getMissingDependencies().get(1), + "${coord:latest(0)} -> file://" + getTestCaseDir() + "/input-data/e/${YEAR}/${DAY}"); } public void testCoordActionPushDependencyMissing() throws Exception { @@ -130,14 +131,14 @@ public class TestCoordActionMissingDependenciesXCommand extends XHCatTestCase { IOUtils.copyCharStream(reader, writer); conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); conf.set(OozieClient.USER_NAME, getTestUser()); - String datasetPrefix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa"; - String datasetSuffix = "hcat://" + getMetastoreAuthority() + "/"; + String datasetSuffix = "/" + TABLE + "/dt=${YEAR}${DAY};country=usa"; + String datasetPrefix = "hcat://" + getMetastoreAuthority() + "/"; - conf.set("data_set_a", datasetSuffix.toString() + DB_A + datasetPrefix); - conf.set("data_set_b", datasetSuffix.toString() + DB_B + datasetPrefix); - conf.set("data_set_c", datasetSuffix.toString() + DB_C + datasetPrefix); - conf.set("data_set_d", datasetSuffix.toString() + DB_D + datasetPrefix); - conf.set("data_set_e", datasetSuffix.toString() + DB_E + datasetPrefix); + conf.set("data_set_a", datasetPrefix.toString() + DB_A + datasetSuffix); + conf.set("data_set_b", datasetPrefix.toString() + DB_B + datasetSuffix); + conf.set("data_set_c", datasetPrefix.toString() + DB_C + datasetSuffix); + conf.set("data_set_d", datasetPrefix.toString() + DB_D + datasetSuffix); + conf.set("data_set_e", datasetPrefix.toString() + DB_E + datasetSuffix); CoordSubmitXCommand sc = new CoordSubmitXCommand(conf); String jobId = sc.call(); @@ -145,13 +146,16 @@ public class TestCoordActionMissingDependenciesXCommand extends XHCatTestCase { List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> data = new CoordActionMissingDependenciesXCommand( jobId + "@1").call(); - assertEquals("${coord:latestRange(-5,0)}", CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst())); + assertEquals(datasetPrefix + "db_a/table1/dt=200901;country=usa", + CoordCommandUtils.getFirstMissingDependency(data.get(0).getFirst())); Map<String, ActionDependency> dependencyMap = data.get(0).getSecond(); assertEquals(6, dependencyMap.size()); assertEquals(1, dependencyMap.get("A").getMissingDependencies().size()); assertEquals(6, dependencyMap.get("B").getMissingDependencies().size()); assertEquals(1, dependencyMap.get("C").getMissingDependencies().size()); + assertEquals("${coord:latestRange(-5,0)} -> " + datasetPrefix.toString() + DB_C + datasetSuffix, + dependencyMap.get("C").getMissingDependencies().get(0)); assertEquals(1, dependencyMap.get("D").getMissingDependencies().size()); assertEquals(6, dependencyMap.get("E").getMissingDependencies().size()); http://git-wip-us.apache.org/repos/asf/oozie/blob/18694b61/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index f780bb2..ccb845c 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2987 Coord action missing dependencies should show URI template with unresolved dependencies OOZIE-2004 Improve Oozie version info output (Artem Ervits via gezapeti) OOZIE-2854 Oozie should handle transient database problems (andras.piros via gezapeti) OOZIE-2371 Add docs for state transitions for WF Action states (daniel.becker via gezapeti)
