OOZIE-2873 Check El Functions before submitting the coordinator (satishsaley)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f669852b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f669852b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f669852b Branch: refs/heads/oya Commit: f669852b332af78aaf658f69e785fad31a1aa392 Parents: 87ed1b3 Author: satishsaley <satishsa...@apache.org> Authored: Wed May 3 20:17:44 2017 -0700 Committer: satishsaley <satishsa...@apache.org> Committed: Wed May 3 20:17:44 2017 -0700 ---------------------------------------------------------------------- .../command/coord/CoordSubmitXCommand.java | 25 +++++++++ .../command/coord/TestCoordSubmitXCommand.java | 30 ++++++++++ .../resources/coord-invalid-el-function.xml | 35 ++++++++++++ .../resources/coord-invalid-output-instance.xml | 58 ++++++++++++++++++++ .../coord-multiple-input-instance4.xml | 2 +- .../coord-multiple-input-start-instance2.xml | 2 +- .../coord-multiple-output-instance4.xml | 2 +- release-log.txt | 1 + 8 files changed, 152 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java index 969336d..0c50621 100644 --- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java @@ -245,6 +245,7 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { LogUtils.setLogInfo(coordJob); if (!dryrun) { + verifyElFunctions(coordJob); queueMaterializeTransitionXCommand(jobId); } else { @@ -1348,4 +1349,28 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand { @Override public void performWrites() throws CommandException { } + + /** + * Verify el functions using dryrun. Throws an exception if dry run throws + * an exception and marks the coordJob as FAILED. + * + * @param coordJob + * @throws Exception + */ + private void verifyElFunctions(CoordinatorJobBean coordJob) throws Exception { + Job.Status prevStatus = coordJob.getStatus(); + try { + // dry run will catch issues with el functions if any + getDryRun(coordJob); + } + catch (Exception e) { + prevStatus = CoordinatorJob.Status.FAILED; + throw e; + } + finally { + coordJob.setStatus(prevStatus); + coordJob.resetPending(); + } + } + } http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java index dcc97ad..91b3792 100644 --- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java +++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java @@ -20,6 +20,7 @@ package org.apache.oozie.command.coord; import java.io.File; import java.io.FileWriter; +import java.io.IOException; import java.io.Reader; import java.io.Writer; import java.net.URI; @@ -1569,4 +1570,33 @@ public class TestCoordSubmitXCommand extends XDataTestCase { .contains("tmp/coord/b/2009/03")); } + public void testFailureOnSubmitWithInvalidInstance() throws Exception { + testFailure("coord-invalid-output-instance.xml"); + } + + public void testFailureOnSubmitWithInvalidElFunction() throws Exception { + testFailure("coord-invalid-el-function.xml"); + } + + private void testFailure(String coordinatorXml) throws IOException { + Configuration conf = new XConfiguration(); + File appPathFile = new File(getTestCaseDir(), "coordinator.xml"); + + Reader reader = IOUtils.getResourceAsReader(coordinatorXml, -1); + Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml")); + IOUtils.copyCharStream(reader, writer); + + conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + CoordSubmitXCommand sc = new CoordSubmitXCommand(conf); + + try { + sc.call(); + fail("Should throw an exception"); + } + catch (CommandException e) { + assertEquals(Job.Status.FAILED, sc.getJob().getStatus()); + assertEquals(e.getErrorCode(), ErrorCode.E0803); + } + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-invalid-el-function.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-invalid-el-function.xml b/core/src/test/resources/coord-invalid-el-function.xml new file mode 100644 index 0000000..11dc478 --- /dev/null +++ b/core/src/test/resources/coord-invalid-el-function.xml @@ -0,0 +1,35 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="${coord:days(1)}" + start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" timezone="UTC"> + <controls> + <concurrency>2</concurrency> + <execution>LIFO</execution> + </controls> + <action> + <workflow> + <app-path>hdfs:///tmp/workflows/</app-path> + <configuration> + <property> + <name>yyyymmdd</name> + <value>${coord:formatTime(coord:nominalTime(), 'DAY')}</value> + </property> + </configuration> + </workflow> + </action> +</coordinator-app> http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-invalid-output-instance.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-invalid-output-instance.xml b/core/src/test/resources/coord-invalid-output-instance.xml new file mode 100644 index 0000000..a90706e --- /dev/null +++ b/core/src/test/resources/coord-invalid-output-instance.xml @@ -0,0 +1,58 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="${coord:days(1)}" + start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" timezone="UTC"> + <controls> + <concurrency>2</concurrency> + <execution>LIFO</execution> + </controls> + <datasets> + <dataset name="a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC"> + <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> + </dataset> + <dataset name="local_a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC"> + <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template> + </dataset> + </datasets> + <input-events> + <data-in name="A" dataset="a"> + <instance>${coord:future(0, 1)}</instance> + </data-in> + </input-events> + <!-- Invalid coordinator application coord:latest()/future(), not permitted in output-events --> + <output-events> + <data-out name="LOCAL_A" dataset="local_a"> + <instance>${coord:future(0,2)}</instance> + </data-out> + </output-events> + <action> + <workflow> + <app-path>hdfs:///tmp/workflows/</app-path> + <configuration> + <property> + <name>inputA</name> + <value>${coord:dataIn('A')}</value> + </property> + <property> + <name>inputB</name> + <value>${coord:dataOut('LOCAL_A')}</value> + </property> + </configuration> + </workflow> + </action> +</coordinator-app> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-multiple-input-instance4.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-multiple-input-instance4.xml b/core/src/test/resources/coord-multiple-input-instance4.xml index 023df26..e092465 100644 --- a/core/src/test/resources/coord-multiple-input-instance4.xml +++ b/core/src/test/resources/coord-multiple-input-instance4.xml @@ -35,7 +35,7 @@ </input-events> <output-events> <data-out name="LOCAL_A" dataset="local_a"> - <instance>${coord:future(0,2)}</instance> + <instance>${coord:current(0)}</instance> </data-out> </output-events> <action> http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-multiple-input-start-instance2.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-multiple-input-start-instance2.xml b/core/src/test/resources/coord-multiple-input-start-instance2.xml index 83d3773..6865d39 100644 --- a/core/src/test/resources/coord-multiple-input-start-instance2.xml +++ b/core/src/test/resources/coord-multiple-input-start-instance2.xml @@ -36,7 +36,7 @@ </input-events> <output-events> <data-out name="LOCAL_A" dataset="local_a"> - <instance>${coord:future(0,2)}</instance> + <instance>${coord:current(0)}</instance> </data-out> </output-events> <action> http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/core/src/test/resources/coord-multiple-output-instance4.xml ---------------------------------------------------------------------- diff --git a/core/src/test/resources/coord-multiple-output-instance4.xml b/core/src/test/resources/coord-multiple-output-instance4.xml index 32d0786..9ed4e40 100644 --- a/core/src/test/resources/coord-multiple-output-instance4.xml +++ b/core/src/test/resources/coord-multiple-output-instance4.xml @@ -35,7 +35,7 @@ </input-events> <output-events> <data-out name="LOCAL_A" dataset="local_a"> - <instance>${coord:formatTime(coord:current(0),'yyyy-MM-dd')}</instance> + <instance>${coord:current(0)}</instance> </data-out> </output-events> <action> http://git-wip-us.apache.org/repos/asf/oozie/blob/f669852b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index d998b2d..944e623 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2873 Check El Functions before submitting the coordinator (satishsaley) OOZIE-2870 non working examples in oozie documentation coordinator spec (andras.piros via pbacsko) OOZIE-2827 amend More directly view of the coordinatorâs history from perspective of workflow action. (Alonzo Zhou via pbacsko) OOZIE-2851 spelling mistakes in examples (Artem Ervits via gezapeti)