Repository: oozie Updated Branches: refs/heads/master 8013a945f -> 6b9ed571e
OOZIE-3389 Getting input dependency list on the UI throws NPE (andras.piros via asalamon74, kmarton) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/6b9ed571 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/6b9ed571 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/6b9ed571 Branch: refs/heads/master Commit: 6b9ed571e599ca7edb3827f9e002660f9b769b3c Parents: 8013a94 Author: Julia Kinga Marton <kmar...@apache.org> Authored: Mon Dec 3 08:42:26 2018 +0100 Committer: Julia Kinga Marton <kmar...@apache.org> Committed: Mon Dec 3 08:42:26 2018 +0100 ---------------------------------------------------------------------- .../CoordActionMissingDependenciesXCommand.java | 15 +- .../dependency/CoordOldInputDependency.java | 5 + .../dependency/TestCoordOldInputDependency.java | 154 +++++++++++++++++++ .../apache/oozie/servlet/TestV2JobServlet.java | 25 +-- release-log.txt | 1 + 5 files changed, 184 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/6b9ed571/core/src/main/java/org/apache/oozie/command/coord/CoordActionMissingDependenciesXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordActionMissingDependenciesXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordActionMissingDependenciesXCommand.java index d37cfe5..f5d8782 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordActionMissingDependenciesXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordActionMissingDependenciesXCommand.java @@ -107,19 +107,20 @@ public class CoordActionMissingDependenciesXCommand @Override protected List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> execute() throws CommandException { - List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> inputDependenciesListPair = + final List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> inputDependenciesListPair = new ArrayList<Pair<CoordinatorActionBean, Map<String, ActionDependency>>>(); try { - for (CoordinatorActionBean coordAction : coordActions) { - CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies(); - CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies(); - Map<String, ActionDependency> dependencyMap = new HashMap<String, ActionDependency>(); + for (final CoordinatorActionBean coordAction : coordActions) { + final CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies(); + final CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies(); + final Map<String, ActionDependency> dependencyMap = new HashMap<>(); dependencyMap.putAll(coordPullInputDependency.getMissingDependencies(coordAction)); dependencyMap.putAll(coordPushInputDependency.getMissingDependencies(coordAction)); - inputDependenciesListPair.add( - new Pair<CoordinatorActionBean, Map<String, ActionDependency>>(coordAction, dependencyMap)); + if (!dependencyMap.isEmpty()) { + inputDependenciesListPair.add(new Pair<>(coordAction, dependencyMap)); + } } } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/oozie/blob/6b9ed571/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java b/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java index 56aef1c..af298b9 100644 --- a/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java +++ b/core/src/main/java/org/apache/oozie/coord/input/dependency/CoordOldInputDependency.java @@ -370,6 +370,11 @@ public class CoordOldInputDependency implements CoordInputDependency { Element eAction = XmlUtils.parseXml(coordAction.getActionXml()); Element inputList = eAction.getChild("input-events", eAction.getNamespace()); + + if (inputList == null || inputList.getChildren().isEmpty()) { + return dependenciesMap; + } + List<Element> eDataEvents = inputList.getChildren("data-in", eAction.getNamespace()); for (Element event : eDataEvents) { Element uri = event.getChild("uris", event.getNamespace()); http://git-wip-us.apache.org/repos/asf/oozie/blob/6b9ed571/core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordOldInputDependency.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordOldInputDependency.java b/core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordOldInputDependency.java new file mode 100644 index 0000000..c6bd3c0 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/coord/input/dependency/TestCoordOldInputDependency.java @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.coord.input.dependency; + +import org.apache.oozie.CoordinatorActionBean; +import org.apache.oozie.client.CoordinatorAction; +import org.apache.oozie.command.CommandException; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XHCatTestCase; +import org.jdom.JDOMException; + +import java.io.IOException; + +public class TestCoordOldInputDependency extends XHCatTestCase { + private Services services; + + @Override + protected void setUp() throws Exception { + super.setUp(); + services = super.setupServicesForHCatalog(); + services.init(); + } + + @Override + protected void tearDown() throws Exception { + services.destroy(); + super.tearDown(); + } + + public void testNoMissingInputDependencies() throws JDOMException, IOException, CommandException { + final CoordinatorActionBean actionWithoutInputDependencies = createActionWithoutInputDependencies(); + + assertEquals("there should be no missing dependencies", + 0, + actionWithoutInputDependencies + .getPullInputDependencies() + .getMissingDependencies(actionWithoutInputDependencies).size()); + } + + public void testOneMissingInputDependency() throws JDOMException, IOException, CommandException { + final CoordinatorActionBean actionWithInputDependencies = createActionWithInputDependencies(); + + assertEquals("there should be one missing dependency", + 1, + actionWithInputDependencies + .getPullInputDependencies() + .getMissingDependencies(actionWithInputDependencies).size()); + } + + private CoordinatorActionBean createActionWithoutInputDependencies() { + final CoordinatorActionBean coordinatorAction = createAction(); + + coordinatorAction.setActionXml("<coordinator-app xmlns=\"uri:oozie:coordinator:0.2\" name=\"cron-coord\"" + + " frequency=\"0/10 * * * *\" timezone=\"UTC\" freq_timeunit=\"CRON\" end_of_duration=\"NONE\"" + + " instance-number=\"1\" action-nominal-time=\"2010-01-01T00:00Z\" action-actual-time=\"2018-11-29T12:55Z\">\n" + + " <action>\n" + + " <workflow>\n" + + " <app-path>hdfs://localhost:9000/user/forsage/examples/apps/cron-schedule</app-path>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>resourceManager</name>\n" + + " <value>localhost:8032</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>nameNode</name>\n" + + " <value>hdfs://localhost:9000</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>queueName</name>\n" + + " <value>default</value>\n" + + " </property>\n" + + " </configuration>\n" + + " </workflow>\n" + + " </action>\n" + + "</coordinator-app>"); + + return coordinatorAction; + } + + private CoordinatorActionBean createAction() { + final CoordinatorActionBean coordinatorAction = new CoordinatorActionBean(); + + coordinatorAction.setId("0000001-181129135145907-oozie-fors-C@1"); + coordinatorAction.setJobId("0000001-181129135145907-oozie-fors-C"); + coordinatorAction.setStatus(CoordinatorAction.Status.SUCCEEDED); + coordinatorAction.setExternalId("0000002-181129135145907-oozie-fors-W"); + + return coordinatorAction; + } + + private CoordinatorActionBean createActionWithInputDependencies() throws IOException { + final CoordinatorActionBean action = createAction(); + + action.setActionXml("<coordinator-app xmlns=\"uri:oozie:coordinator:0.2\" name=\"cron-coord\"" + + " frequency=\"0/10 * * * *\" timezone=\"UTC\" freq_timeunit=\"CRON\" end_of_duration=\"NONE\"" + + " instance-number=\"1\" action-nominal-time=\"2010-01-01T00:00Z\" action-actual-time=\"2018-11-29T12:55Z\">\n" + + " <datasets>\n" + + " <dataset name=\"data-1\" frequency=\"${coord:minutes(20)}\" initial-instance=\"2010-01-01T00:00Z\">\n" + + " <uri-template>${nameNode}/${YEAR}/${MONTH}/${DAY}/${HOUR}/${MINUTE}</uri-template>\n" + + " </dataset>\n" + + " </datasets>\n" + + " <input-events>\n" + + " <data-in name=\"input-1\" dataset=\"data-1\">\n" + + " <start-instance>${coord:current(-2)}</start-instance>\n" + + " <end-instance>${coord:current(0)}</end-instance>\n" + + " </data-in>\n" + + " </input-events>\n" + + " <action>\n" + + " <workflow>\n" + + " <app-path>hdfs://localhost:9000/user/forsage/examples/apps/cron-schedule</app-path>\n" + + " <configuration>\n" + + " <property>\n" + + " <name>resourceManager</name>\n" + + " <value>localhost:8032</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>nameNode</name>\n" + + " <value>hdfs://localhost:9000</value>\n" + + " </property>\n" + + " <property>\n" + + " <name>queueName</name>\n" + + " <value>default</value>\n" + + " </property>\n" + + " </configuration>\n" + + " </workflow>\n" + + " </action>\n" + + "</coordinator-app>"); + + final CoordInputDependency coordPullInputDependency = CoordInputDependencyFactory + .createPullInputDependencies(true); + coordPullInputDependency.addUnResolvedList("data-1", "data-1"); + + action.setMissingDependencies(coordPullInputDependency.serialize()); + action.setPullInputDependencies(coordPullInputDependency); + + return action; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/6b9ed571/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java b/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java index bacfe89..b42421d 100644 --- a/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java +++ b/core/src/test/java/org/apache/oozie/servlet/TestV2JobServlet.java @@ -19,7 +19,6 @@ package org.apache.oozie.servlet; import org.apache.oozie.client.CoordinatorWfAction; -import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.service.ConfigurationService; @@ -117,13 +116,20 @@ public class TestV2JobServlet extends DagServletTestCase { url = createURL(MockCoordinatorEngineService.JOB_ID + (MockCoordinatorEngineService.coordJobs.size() + 1), params); conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("GET"); - assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertBadRequestOrInternalServerError(conn.getResponseCode()); assertEquals(RestConstants.JOB_SHOW_INFO, MockCoordinatorEngineService.did); return null; } }); } + private void assertBadRequestOrInternalServerError(final int responseCode) { + assertTrue(String.format("HTTP response code [%d] is unexpected, should be one of [%d, %d]", + responseCode, HttpServletResponse.SC_BAD_REQUEST, HttpServletResponse.SC_INTERNAL_SERVER_ERROR), + HttpServletResponse.SC_BAD_REQUEST == responseCode + || HttpServletResponse.SC_INTERNAL_SERVER_ERROR == responseCode); + } + public void testGetCoordActionReruns() throws Exception { runTest("/v2/job/*", V1JobServlet.class, IS_SECURITY_ENABLED, new Callable<Void>() { @Override @@ -180,7 +186,7 @@ public class TestV2JobServlet extends DagServletTestCase { conn.setRequestMethod("PUT"); conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); conn.setDoOutput(true); - assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertBadRequestOrInternalServerError(conn.getResponseCode()); assertEquals(RestConstants.JOB_ACTION_CHANGE, MockCoordinatorEngineService.did); return null; @@ -219,7 +225,7 @@ public class TestV2JobServlet extends DagServletTestCase { conn.setRequestMethod("PUT"); conn.setRequestProperty("content-type", RestConstants.XML_CONTENT_TYPE); conn.setDoOutput(true); - assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertBadRequestOrInternalServerError(conn.getResponseCode()); assertEquals(RestConstants.JOB_ACTION_IGNORE, MockCoordinatorEngineService.did); return null; @@ -313,7 +319,7 @@ public class TestV2JobServlet extends DagServletTestCase { URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("GET"); - assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertBadRequestOrInternalServerError(conn.getResponseCode()); return null; } @@ -334,7 +340,7 @@ public class TestV2JobServlet extends DagServletTestCase { URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("GET"); - assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertBadRequestOrInternalServerError(conn.getResponseCode()); return null; } @@ -355,7 +361,7 @@ public class TestV2JobServlet extends DagServletTestCase { URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("GET"); - assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertBadRequestOrInternalServerError(conn.getResponseCode()); return null; } @@ -419,7 +425,7 @@ public class TestV2JobServlet extends DagServletTestCase { URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("GET"); - assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertBadRequestOrInternalServerError(conn.getResponseCode()); return null; } @@ -440,7 +446,8 @@ public class TestV2JobServlet extends DagServletTestCase { URL url = createURL(MockCoordinatorEngineService.JOB_ID + 1, params); HttpURLConnection conn = (HttpURLConnection) url.openConnection(); conn.setRequestMethod("GET"); - assertEquals(HttpServletResponse.SC_BAD_REQUEST, conn.getResponseCode()); + assertBadRequestOrInternalServerError(conn.getResponseCode()); + return null; } }); http://git-wip-us.apache.org/repos/asf/oozie/blob/6b9ed571/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index b0dd230..5690017 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.2.0 release (trunk - unreleased) +OOZIE-3389 Getting input dependency list on the UI throws NPE (andras.piros via asalamon74, kmarton) OOZIE-3382 [SSH action] [SSH action] Optimize process streams draining (asalamon74 via andras.piros) OOZIE-3384 [tests] TestWorkflowActionRetryInfoXCommand#testRetryConsoleUrlForked() is flaky (asalamon74 via kmarton) OOZIE-3386 Misleading error message when workflow application does not exist (kmarton)
