OOZIE-2691 Show workflow action retry information in UI
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/bf2802e4 Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/bf2802e4 Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/bf2802e4 Branch: refs/heads/oya Commit: bf2802e4052ad9612bb63b41671f32fe9237f496 Parents: c58f545 Author: puru <puru.s...@gmail.com> Authored: Sun Jan 22 16:13:59 2017 -0800 Committer: puru <puru.s...@gmail.com> Committed: Sun Jan 22 16:13:59 2017 -0800 ---------------------------------------------------------------------- .../java/org/apache/oozie/cli/OozieCLI.java | 35 ++- .../org/apache/oozie/client/OozieClient.java | 39 ++- .../org/apache/oozie/client/rest/JsonTags.java | 2 +- .../apache/oozie/client/rest/RestConstants.java | 2 + .../main/java/org/apache/oozie/DagEngine.java | 17 ++ .../oozie/command/wf/ActionCheckXCommand.java | 2 +- .../oozie/command/wf/ActionEndXCommand.java | 2 +- .../oozie/command/wf/ActionStartXCommand.java | 9 +- .../apache/oozie/command/wf/ActionXCommand.java | 39 ++- .../wf/WorkflowActionRetryInfoXCommand.java | 115 ++++++++ .../apache/oozie/servlet/BaseJobServlet.java | 22 +- .../org/apache/oozie/servlet/V0JobServlet.java | 7 + .../org/apache/oozie/servlet/V1JobServlet.java | 8 +- .../org/apache/oozie/servlet/V2JobServlet.java | 20 +- .../java/org/apache/oozie/util/JobUtils.java | 10 + .../apache/oozie/ForTestingActionExecutor.java | 16 +- .../org/apache/oozie/client/TestOozieCLI.java | 15 ++ .../wf/TestWorkflowActionRetryInfoXCommand.java | 138 ++++++++++ .../oozie/servlet/MockDagEngineService.java | 6 +- core/src/test/resources/wf-ext-schema.xsd | 1 + docs/src/site/twiki/DG_CommandLineTool.twiki | 29 ++ docs/src/site/twiki/WebServicesAPI.twiki | 31 +++ release-log.txt | 1 + webapp/src/main/webapp/oozie-console.js | 270 ++++++++++++++++--- 24 files changed, 768 insertions(+), 68 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/client/src/main/java/org/apache/oozie/cli/OozieCLI.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java index e67fae9..6e30d7e 100644 --- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java +++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java @@ -179,6 +179,8 @@ public class OozieCLI { public static final String ALL_WORKFLOWS_FOR_COORD_ACTION = "allruns"; + public static final String WORKFLOW_ACTIONS_RETRIES = "retries"; + private static final String[] OOZIE_HELP = { "the env variable '" + ENV_OOZIE_URL + "' is used as default value for the '-" + OOZIE_OPTION + "' option", "the env variable '" + ENV_OOZIE_TIME_ZONE + "' is used as default value for the '-" + TIME_ZONE_OPTION + "' option", @@ -381,9 +383,11 @@ public class OozieCLI { Option slaChange = new Option(SLA_CHANGE, true, "Update sla param for jobs, supported param are should-start, should-end, nominal-time and max-duration"); - Option doAs = new Option(DO_AS_OPTION, true, "doAs user, impersonates as the specified user"); + Option workflowActionRetries = new Option(WORKFLOW_ACTIONS_RETRIES, true, + "Get information of the retry attempts for a given workflow action"); + OptionGroup actions = new OptionGroup(); actions.addOption(submit); actions.addOption(start); @@ -406,7 +410,7 @@ public class OozieCLI { actions.addOption(slaDisableAlert); actions.addOption(slaEnableAlert); actions.addOption(slaChange); - + actions.addOption(workflowActionRetries); actions.setRequired(true); Options jobOptions = new Options(); jobOptions.addOption(oozie); @@ -1184,6 +1188,7 @@ public class OozieCLI { } printWorkflowAction(wc.getWorkflowActionInfo(optionValue), timeZoneId, options.contains(VERBOSE_OPTION)); + } else { String filter = commandLine.getOptionValue(FILTER_OPTION); @@ -1319,6 +1324,12 @@ public class OozieCLI { else if (options.contains(SLA_CHANGE)) { slaAlertCommand(commandLine.getOptionValue(SLA_CHANGE), wc, commandLine, options); } + else if (options.contains(WORKFLOW_ACTIONS_RETRIES)) { + printWorkflowActionRetries( + wc.getWorkflowActionRetriesInfo(commandLine.getOptionValue(WORKFLOW_ACTIONS_RETRIES)), + commandLine.getOptionValue(WORKFLOW_ACTIONS_RETRIES)); + } + } catch (OozieClientException ex) { throw new OozieCLIException(ex.toString(), ex); @@ -1484,6 +1495,26 @@ public class OozieCLI { System.out.println(RULER); } + void printWorkflowActionRetries(List<Map<String, String>> retries, String actionId) { + System.out.println("ID : " + maskIfNull(actionId)); + if (retries.isEmpty()) { + System.out.println("No Retries"); + } + for (Map<String, String> retry: retries) { + System.out.println(RULER); + System.out.println("Attempt : " + retry.get(JsonTags.ACTION_ATTEMPT)); + System.out.println("Start Time : " + retry.get(JsonTags.WORKFLOW_ACTION_START_TIME)); + System.out.println("End Time : " + retry.get(JsonTags.WORKFLOW_ACTION_END_TIME)); + if (null != retry.get(JsonTags.WORKFLOW_ACTION_CONSOLE_URL)) { + System.out.println("Console URL : " + retry.get(JsonTags.WORKFLOW_ACTION_CONSOLE_URL)); + } + if (null != retry.get(JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS)) { + System.out.println("Child URL : " + retry.get(JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS)); + } + } + System.out.println(RULER); + } + private static final String WORKFLOW_JOBS_FORMATTER = "%-41s%-13s%-10s%-10s%-10s%-24s%-24s"; private static final String COORD_JOBS_FORMATTER = "%-41s%-15s%-10s%-5s%-13s%-24s%-24s"; private static final String BUNDLE_JOBS_FORMATTER = "%-41s%-15s%-10s%-20s%-20s%-13s%-13s"; http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/client/src/main/java/org/apache/oozie/client/OozieClient.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java index 12c80cb..a107c4a 100644 --- a/client/src/main/java/org/apache/oozie/client/OozieClient.java +++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java @@ -47,7 +47,6 @@ import java.io.Reader; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -59,6 +58,9 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; import java.util.concurrent.Callable; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.type.TypeReference; + /** * Client API to submit and manage Oozie workflow jobs against an Oozie intance. @@ -1025,6 +1027,29 @@ public class OozieClient { } } + private class WorkflowActionRetriesInfo extends ClientCallable<List<Map<String, String>>> { + WorkflowActionRetriesInfo(String actionId) { + super("GET", RestConstants.JOB, notEmpty(actionId, "id"), + prepareParams(RestConstants.JOB_SHOW_PARAM, RestConstants.JOB_SHOW_ACTION_RETRIES_PARAM)); + } + + @Override + protected List<Map<String, String>> call(HttpURLConnection conn) + throws IOException, OozieClientException { + if ((conn.getResponseCode() == HttpURLConnection.HTTP_OK)) { + Reader reader = new InputStreamReader(conn.getInputStream()); + JSONObject json = (JSONObject) JSONValue.parse(reader); + return new ObjectMapper().readValue(json.get(JsonTags.WORKFLOW_ACTION_RETRIES).toString(), + new TypeReference<List<Map<String, String>>>() { + }); + } + else { + handleError(conn); + } + return null; + } + } + /** * Get the info of a workflow job. * @@ -1069,6 +1094,18 @@ public class OozieClient { return new WorkflowActionInfo(actionId).call(); } + + /** + * Get the info of a workflow action. + * + * @param actionId Id. + * @return the workflow action retries info. + * @throws OozieClientException thrown if the job info could not be retrieved. + */ + public List<Map<String, String>> getWorkflowActionRetriesInfo(String actionId) throws OozieClientException { + return new WorkflowActionRetriesInfo(actionId).call(); + } + /** * Get the log of a workflow job. * http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java b/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java index 397e9ed..d670142 100644 --- a/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java +++ b/client/src/main/java/org/apache/oozie/client/rest/JsonTags.java @@ -244,7 +244,7 @@ public interface JsonTags { String COORD_UPDATE_DIFF = "diff"; String STATUS = "status"; - + String ACTION_ATTEMPT = "attempt"; String VALIDATE = "validate"; } http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java ---------------------------------------------------------------------- diff --git a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java index 4129364..9a3be97 100644 --- a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java +++ b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java @@ -81,6 +81,8 @@ public interface RestConstants { String JOB_SHOW_PARAM = "show"; + String JOB_SHOW_ACTION_RETRIES_PARAM = "retries"; + String JOB_SHOW_CONFIG = "config"; String JOB_SHOW_INFO = "info"; http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/DagEngine.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/DagEngine.java b/core/src/main/java/org/apache/oozie/DagEngine.java index 7597142..57d2761 100644 --- a/core/src/main/java/org/apache/oozie/DagEngine.java +++ b/core/src/main/java/org/apache/oozie/DagEngine.java @@ -55,6 +55,7 @@ import org.apache.oozie.command.wf.SubmitSqoopXCommand; import org.apache.oozie.command.wf.SubmitXCommand; import org.apache.oozie.command.wf.SuspendXCommand; import org.apache.oozie.command.wf.WorkflowActionInfoXCommand; +import org.apache.oozie.command.wf.WorkflowActionRetryInfoXCommand; import org.apache.oozie.executor.jpa.JPAExecutorException; import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor; import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery; @@ -583,6 +584,22 @@ public class DagEngine extends BaseEngine { } } + /** + * Gets the workflow action retries. + * + * @param actionId the action id + * @return the workflow action retries + * @throws BaseEngineException the base engine exception + */ + public List<Map<String, String>> getWorkflowActionRetries(String actionId) throws BaseEngineException { + try { + return new WorkflowActionRetryInfoXCommand(actionId).call(); + } + catch (CommandException ex) { + throw new BaseEngineException(ex); + } + } + /* (non-Javadoc) * @see org.apache.oozie.BaseEngine#dryRunSubmit(org.apache.hadoop.conf.Configuration) */ http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java index d0551ff..335527d 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/ActionCheckXCommand.java @@ -210,7 +210,7 @@ public class ActionCheckXCommand extends ActionXCommand<Void> { switch (ex.getErrorType()) { case ERROR: // If allowed to retry, this will handle it; otherwise, we should fall through to FAILED - if (handleUserRetry(wfAction, wfJob)) { + if (handleUserRetry(context, wfAction)) { break; } case FAILED: http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/command/wf/ActionEndXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/ActionEndXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/ActionEndXCommand.java index 740b8d3..86ee1cc 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/ActionEndXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/ActionEndXCommand.java @@ -216,7 +216,7 @@ public class ActionEndXCommand extends ActionXCommand<Void> { shouldHandleUserRetry = true; break; } - if (!shouldHandleUserRetry || !handleUserRetry(wfAction, wfJob)) { + if (!shouldHandleUserRetry || !handleUserRetry(context, wfAction)) { SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfAction.getSlaXml(), wfAction.getId(), slaStatus, SlaAppType.WORKFLOW_ACTION); if(slaEvent != null) { insertList.add(slaEvent); http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/command/wf/ActionStartXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/ActionStartXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/ActionStartXCommand.java index edfac48..5bc0eab 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/ActionStartXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/ActionStartXCommand.java @@ -40,6 +40,7 @@ import org.apache.oozie.client.WorkflowJob; import org.apache.oozie.client.SLAEvent.SlaAppType; import org.apache.oozie.client.SLAEvent.Status; import org.apache.oozie.client.rest.JsonBean; +import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.command.CommandException; import org.apache.oozie.command.PreconditionException; import org.apache.oozie.command.XCommand; @@ -55,8 +56,10 @@ import org.apache.oozie.service.EventHandlerService; import org.apache.oozie.service.JPAService; import org.apache.oozie.service.Services; import org.apache.oozie.service.UUIDService; +import org.apache.oozie.util.DateUtils; import org.apache.oozie.util.ELEvaluationException; import org.apache.oozie.util.Instrumentation; +import org.apache.oozie.util.JobUtils; import org.apache.oozie.util.LogUtils; import org.apache.oozie.util.XLog; import org.apache.oozie.util.XmlUtils; @@ -230,7 +233,11 @@ public class ActionStartXCommand extends ActionXCommand<org.apache.oozie.command Instrumentation.Cron cron = new Instrumentation.Cron(); cron.start(); - context.setStartTime(); + // do not override starttime for retries + if (wfAction.getStartTime() == null) { + context.setStartTime(); + } + context.setVar(JobUtils.getRetryKey(wfAction, JsonTags.WORKFLOW_ACTION_START_TIME), String.valueOf(new Date().getTime())); executor.start(context, wfAction); cron.stop(); FaultInjection.activate("org.apache.oozie.command.SkipCommitFaultInjection"); http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java index 836e5d4..4f127c0 100644 --- a/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java +++ b/core/src/main/java/org/apache/oozie/command/wf/ActionXCommand.java @@ -39,6 +39,7 @@ import org.apache.oozie.action.ActionExecutor; import org.apache.oozie.client.Job; import org.apache.oozie.client.WorkflowAction; import org.apache.oozie.client.WorkflowJob; +import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.command.CommandException; import org.apache.oozie.service.CallbackService; import org.apache.oozie.service.ConfigurationService; @@ -51,6 +52,7 @@ import org.apache.oozie.service.Services; import org.apache.oozie.util.ELEvaluator; import org.apache.oozie.util.InstrumentUtils; import org.apache.oozie.util.Instrumentation; +import org.apache.oozie.util.JobUtils; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.workflow.WorkflowException; import org.apache.oozie.workflow.WorkflowInstance; @@ -64,6 +66,7 @@ import org.apache.oozie.workflow.lite.NodeDef; */ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { private static final String INSTRUMENTATION_GROUP = "action.executors"; + public static final String RETRY = "retry."; protected static final String RECOVERY_ID_SEPARATOR = "@"; @@ -157,8 +160,7 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { LOG.warn("Setting Action Status to [{0}]", status); ActionExecutorContext aContext = (ActionExecutorContext) context; WorkflowActionBean action = (WorkflowActionBean) aContext.getAction(); - WorkflowJobBean wfJob = (WorkflowJobBean) context.getWorkflow(); - if (!handleUserRetry(action, wfJob)) { + if (!handleUserRetry(context, action)) { incrActionErrorCounter(action.getType(), "error", 1); action.setPending(); if (isStart) { @@ -192,7 +194,7 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { */ public void failJob(ActionExecutor.Context context, WorkflowActionBean action) throws CommandException { WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow(); - if (!handleUserRetry(action, workflow)) { + if (!handleUserRetry(context, action)) { incrActionErrorCounter(action.getType(), "failed", 1); LOG.warn("Failing Job due to failed action [{0}]", action.getName()); try { @@ -220,7 +222,8 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { * @return true if user-retry has to be handled for this action * @throws CommandException thrown if unable to fail job */ - public boolean handleUserRetry(WorkflowActionBean action, WorkflowJobBean wfJob) throws CommandException { + public boolean handleUserRetry(ActionExecutor.Context context, WorkflowActionBean action) throws CommandException { + WorkflowJobBean wfJob = (WorkflowJobBean) context.getWorkflow(); String errorCode = action.getErrorCode(); Set<String> allowedRetryCode = LiteWorkflowStoreService.getUserRetryErrorCode(); @@ -232,6 +235,7 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { ActionExecutor.RETRYPOLICY retryPolicy = getUserRetryPolicy(action, wfJob); long interval = getRetryDelay(action.getUserRetryCount(), action.getUserRetryInterval() * 60, retryPolicy); action.setStatus(WorkflowAction.Status.USER_RETRY); + context.setVar(JobUtils.getRetryKey(action, JsonTags.WORKFLOW_ACTION_END_TIME), String.valueOf(new Date().getTime())); action.incrmentUserRetryCount(); action.setPending(); queue(new ActionStartXCommand(action.getId(), action.getType()), interval); @@ -295,22 +299,29 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { private Job.Status jobStatus; /** - * Constructing the ActionExecutorContext, setting the private members - * and constructing the proto configuration - */ - public ActionExecutorContext(WorkflowJobBean workflow, WorkflowActionBean action, boolean isRetry, boolean isUserRetry) { + * Constructing the ActionExecutorContext, setting the private members + * and constructing the proto configuration + */ + public ActionExecutorContext(WorkflowJobBean workflow, WorkflowActionBean action, boolean isRetry, + boolean isUserRetry) { this.workflow = workflow; this.action = action; this.isRetry = isRetry; this.isUserRetry = isUserRetry; - try { - protoConf = new XConfiguration(new StringReader(workflow.getProtoActionConf())); - } - catch (IOException ex) { - throw new RuntimeException("It should not happen", ex); + if (null != workflow.getProtoActionConf()) { + try { + protoConf = new XConfiguration(new StringReader(workflow.getProtoActionConf())); + } + catch (IOException ex) { + throw new RuntimeException("It should not happen", ex); + } } } + public ActionExecutorContext(WorkflowJobBean workflow, WorkflowActionBean action) { + this(workflow, action, false, false); + } + /* * (non-Javadoc) * @see org.apache.oozie.action.ActionExecutor.Context#getCallbackUrl(java.lang.String) @@ -388,6 +399,7 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { * @see org.apache.oozie.action.ActionExecutor.Context#setStartData(java.lang.String, java.lang.String, java.lang.String) */ public void setStartData(String externalId, String trackerUri, String consoleUrl) { + setVar(JobUtils.getRetryKey(action, JsonTags.WORKFLOW_ACTION_CONSOLE_URL), consoleUrl); action.setStartData(externalId, trackerUri, consoleUrl); started = true; } @@ -423,6 +435,7 @@ public abstract class ActionXCommand<T> extends WorkflowXCommand<T> { * @see org.apache.oozie.action.ActionExecutor.Context#setExternalChildIDs(java.lang.String) */ public void setExternalChildIDs(String externalChildIDs) { + setVar(JobUtils.getRetryKey(action, JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS), externalChildIDs); action.setExternalChildIDs(externalChildIDs); executed = true; } http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/command/wf/WorkflowActionRetryInfoXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/command/wf/WorkflowActionRetryInfoXCommand.java b/core/src/main/java/org/apache/oozie/command/wf/WorkflowActionRetryInfoXCommand.java new file mode 100644 index 0000000..fac0989 --- /dev/null +++ b/core/src/main/java/org/apache/oozie/command/wf/WorkflowActionRetryInfoXCommand.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.command.wf; + +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.oozie.ErrorCode; +import org.apache.oozie.WorkflowActionBean; +import org.apache.oozie.WorkflowJobBean; +import org.apache.oozie.client.rest.JsonTags; +import org.apache.oozie.client.rest.JsonUtils; +import org.apache.oozie.command.CommandException; +import org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext; +import org.apache.oozie.executor.jpa.JPAExecutorException; +import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor; +import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor; +import org.apache.oozie.executor.jpa.WorkflowActionQueryExecutor.WorkflowActionQuery; +import org.apache.oozie.executor.jpa.WorkflowJobQueryExecutor.WorkflowJobQuery; +import org.apache.oozie.util.JobUtils; +import org.apache.oozie.util.LogUtils; + +public class WorkflowActionRetryInfoXCommand extends WorkflowXCommand<List<Map<String, String>>> { + private String actionId; + private WorkflowJobBean wfJob; + protected WorkflowActionBean wfAction = null; + + public WorkflowActionRetryInfoXCommand(String id) { + super("action.retries.info", "action.retries.info", 1); + this.actionId = id; + } + + @Override + protected List<Map<String, String>> execute() throws CommandException { + List<Map<String, String>> retriesList = new ArrayList<Map<String, String>>(); + ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfJob, wfAction); + for (int i = 0; i < wfAction.getUserRetryCount(); i++) { + Map<String, String> retries = new HashMap<String, String>(); + String value = context.getVar(JobUtils.getRetryKey(JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS, i)); + if (value != null) { + retries.put(JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS, value); + } + value = context.getVar(JobUtils.getRetryKey(JsonTags.WORKFLOW_ACTION_CONSOLE_URL, i)); + if (value != null) { + retries.put(JsonTags.WORKFLOW_ACTION_CONSOLE_URL, value); + } + value = context.getVar(JobUtils.getRetryKey(JsonTags.WORKFLOW_ACTION_START_TIME, i)); + if (value != null) { + retries.put(JsonTags.WORKFLOW_ACTION_START_TIME, + JsonUtils.formatDateRfc822(new Date(Long.parseLong(value)))); + } + value = context.getVar(JobUtils.getRetryKey(JsonTags.WORKFLOW_ACTION_END_TIME, i)); + if (value != null) { + retries.put(JsonTags.WORKFLOW_ACTION_END_TIME, + JsonUtils.formatDateRfc822(new Date(Long.parseLong(value)))); + } + retries.put(JsonTags.ACTION_ATTEMPT, String.valueOf(i + 1)); + retriesList.add(retries); + } + return retriesList; + } + + @Override + public String getEntityKey() { + return null; + } + + @Override + protected void loadState() throws CommandException { + try { + this.wfAction = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_CHECK, + actionId); + this.wfJob = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW_DEFINITION, + actionId.substring(0, actionId.indexOf("@"))); + LogUtils.setLogInfo(wfAction); + } + catch (JPAExecutorException ex) { + if (ex.getErrorCode() == ErrorCode.E0605) { + throw new CommandException(ErrorCode.E0605, actionId); + } + else { + throw new CommandException(ex); + } + } + } + + @Override + protected void verifyPrecondition() throws CommandException { + } + + @Override + protected boolean isLockRequired() { + return false; + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java index 2110522..87a2b42 100644 --- a/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/BaseJobServlet.java @@ -41,6 +41,7 @@ import org.apache.oozie.util.ConfigUtils; import org.apache.oozie.util.JobUtils; import org.apache.oozie.util.XConfiguration; import org.apache.oozie.util.XLog; +import org.json.simple.JSONArray; import org.json.simple.JSONObject; public abstract class BaseJobServlet extends JsonRestServlet { @@ -353,8 +354,14 @@ public abstract class BaseJobServlet extends JsonRestServlet { json.put(JsonTags.STATUS, status); startCron(); sendJsonResponse(response, HttpServletResponse.SC_OK, json); - } - else { + } else if (show.equals(RestConstants.JOB_SHOW_ACTION_RETRIES_PARAM)) { + stopCron(); + JSONArray retries = getActionRetries(request, response); + JSONObject json = new JSONObject(); + json.put(JsonTags.WORKFLOW_ACTION_RETRIES, retries); + startCron(); + sendJsonResponse(response, HttpServletResponse.SC_OK, json); + } else { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0303, RestConstants.JOB_SHOW_PARAM, show); } @@ -570,4 +577,15 @@ public abstract class BaseJobServlet extends JsonRestServlet { abstract void slaChange(HttpServletRequest request, HttpServletResponse response) throws XServletException, IOException; + /** + * Gets the action retries. + * + * @param request the request + * @param response the response + * @return the action retries + * @throws XServletException the x servlet exception + * @throws IOException Signals that an I/O exception has occurred. + */ + abstract JSONArray getActionRetries(HttpServletRequest request, HttpServletResponse response) + throws XServletException, IOException; } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java index 86ff278..0c42128 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V0JobServlet.java @@ -29,6 +29,7 @@ import org.apache.oozie.DagEngineException; import org.apache.oozie.client.rest.JsonBean; import org.apache.oozie.service.DagEngineService; import org.apache.oozie.service.Services; +import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.apache.oozie.ErrorCode; @@ -260,4 +261,10 @@ public class V0JobServlet extends BaseJobServlet { void slaChange(HttpServletRequest request, HttpServletResponse response) throws XServletException, IOException { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v0"); } + + @Override + JSONArray getActionRetries(HttpServletRequest request, HttpServletResponse response) throws XServletException, + IOException { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v0"); + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java index 60f1029..95dcca6 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V1JobServlet.java @@ -1123,4 +1123,10 @@ public class V1JobServlet extends BaseJobServlet { void slaChange(HttpServletRequest request, HttpServletResponse response) throws XServletException, IOException { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v1"); } -} + + @Override + JSONArray getActionRetries(HttpServletRequest request, HttpServletResponse response) throws XServletException, + IOException { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0302, "Not supported in v1"); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java b/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java index 662a7ff..3a0ffb0 100644 --- a/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java +++ b/core/src/main/java/org/apache/oozie/servlet/V2JobServlet.java @@ -21,6 +21,7 @@ package org.apache.oozie.servlet; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -29,7 +30,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.oozie.BaseEngine; import org.apache.oozie.BaseEngineException; -import org.apache.oozie.BundleEngine; import org.apache.oozie.CoordinatorActionBean; import org.apache.oozie.CoordinatorActionInfo; import org.apache.oozie.CoordinatorEngine; @@ -42,10 +42,12 @@ import org.apache.oozie.client.rest.JsonBean; import org.apache.oozie.client.rest.JsonTags; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.command.CommandException; +import org.apache.oozie.command.wf.ActionXCommand; import org.apache.oozie.service.BundleEngineService; import org.apache.oozie.service.CoordinatorEngineService; import org.apache.oozie.service.DagEngineService; import org.apache.oozie.service.Services; +import org.json.simple.JSONArray; import org.json.simple.JSONObject; @SuppressWarnings("serial") @@ -295,9 +297,23 @@ public class V2JobServlet extends V1JobServlet { catch (BaseEngineException e) { throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, e); } - } + @SuppressWarnings("unchecked") + @Override + JSONArray getActionRetries(HttpServletRequest request, HttpServletResponse response) + throws XServletException, IOException { + JSONArray jsonArray = new JSONArray(); + String jobId = getResourceName(request); + try { + jsonArray.addAll(Services.get().get(DagEngineService.class).getDagEngine(getUser(request)) + .getWorkflowActionRetries(jobId)); + return jsonArray; + } + catch (BaseEngineException ex) { + throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex); + } + } /** * Gets the base engine based on jobId. http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/main/java/org/apache/oozie/util/JobUtils.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/oozie/util/JobUtils.java b/core/src/main/java/org/apache/oozie/util/JobUtils.java index a7a53b3..63f88ac 100644 --- a/core/src/main/java/org/apache/oozie/util/JobUtils.java +++ b/core/src/main/java/org/apache/oozie/util/JobUtils.java @@ -28,10 +28,12 @@ import org.apache.hadoop.filecache.DistributedCache; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.oozie.ErrorCode; +import org.apache.oozie.WorkflowActionBean; import org.apache.oozie.action.hadoop.JavaActionExecutor; import org.apache.oozie.client.OozieClient; import org.apache.oozie.client.XOozieClient; import org.apache.oozie.command.CommandException; +import org.apache.oozie.command.wf.ActionXCommand; import org.apache.oozie.service.HadoopAccessorException; import org.apache.oozie.service.HadoopAccessorService; import org.apache.oozie.service.Services; @@ -166,4 +168,12 @@ public class JobUtils { DistributedCache.addFileToClassPath(file, conf, fs); } } + + public static String getRetryKey(WorkflowActionBean wfAction, String key) { + return ActionXCommand.RETRY + wfAction.getUserRetryCount() + "." + key; + } + + public static String getRetryKey(String key, int retry) { + return ActionXCommand.RETRY + retry + "." + key; + } } http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/test/java/org/apache/oozie/ForTestingActionExecutor.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/ForTestingActionExecutor.java b/core/src/test/java/org/apache/oozie/ForTestingActionExecutor.java index a70dc02..c9d0c98 100644 --- a/core/src/test/java/org/apache/oozie/ForTestingActionExecutor.java +++ b/core/src/test/java/org/apache/oozie/ForTestingActionExecutor.java @@ -60,19 +60,26 @@ public class ForTestingActionExecutor extends ActionExecutor { throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, TEST_ERROR, "start"); } String externalStatus = eConf.getChild("external-status", ns).getText().trim(); + Element externalChildIds = eConf.getChild("external-childIds", ns); String runningMode = "sync"; Element runningModeElement = eConf.getChild("running-mode", ns); if (null != runningModeElement) { - if (runningModeElement.getText().trim().equals("async")) { - runningMode = "async"; - } + runningMode = runningModeElement.getText().trim(); } if (runningMode.equals("async")) { context.setStartData("blah", "blah", "blah"); return; } + if (runningMode.equals("async-error")) { + context.setStartData("blah", "blah", "blah"); + context.setExecutionData(externalStatus, null); + if (null != externalChildIds) { + context.setExternalChildIDs(externalChildIds.getText()); + } + throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, TEST_ERROR, "start"); + } boolean callSetExecutionData = true; Element setStartData = eConf.getChild("avoid-set-execution-data", ns); if (null != setStartData) { @@ -83,6 +90,9 @@ public class ForTestingActionExecutor extends ActionExecutor { if (callSetExecutionData) { context.setExecutionData(externalStatus, null); } + if (null != externalChildIds) { + context.setExternalChildIDs(externalChildIds.getText()); + } } public void end(Context context, WorkflowAction action) throws ActionExecutorException { http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java index ce95ff3..8ec38e4 100644 --- a/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java +++ b/core/src/test/java/org/apache/oozie/client/TestOozieCLI.java @@ -1154,6 +1154,21 @@ public class TestOozieCLI extends DagServletTestCase { }); } + public void testWfActionRetries() throws Exception { + runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { + @Override + public Void call() throws Exception { + String oozieUrl = getContextURL(); + MockDagEngineService.reset(); + String[] args = new String[] { "job", "-oozie", oozieUrl, "-retries", + MockDagEngineService.JOB_ID + "0" + MockDagEngineService.JOB_ID_END + "@a"}; + assertEquals(0, new OozieCLI().run(args)); + assertEquals(RestConstants.JOB_SHOW_ACTION_RETRIES_PARAM, MockDagEngineService.did); + return null; + } + }); + } + public void testAdminQueueDump() throws Exception { runTest(END_POINTS, SERVLET_CLASSES, IS_SECURITY_ENABLED, new Callable<Void>() { @Override http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionRetryInfoXCommand.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionRetryInfoXCommand.java b/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionRetryInfoXCommand.java new file mode 100644 index 0000000..1a5f354 --- /dev/null +++ b/core/src/test/java/org/apache/oozie/command/wf/TestWorkflowActionRetryInfoXCommand.java @@ -0,0 +1,138 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.oozie.command.wf; + +import java.io.File; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.conf.Configuration; +import org.apache.oozie.ForTestingActionExecutor; +import org.apache.oozie.WorkflowActionBean; +import org.apache.oozie.client.OozieClient; +import org.apache.oozie.client.rest.JsonTags; +import org.apache.oozie.client.rest.JsonUtils; +import org.apache.oozie.executor.jpa.WorkflowActionsGetForJobJPAExecutor; +import org.apache.oozie.service.ActionService; +import org.apache.oozie.service.ExtendedCallableQueueService; +import org.apache.oozie.service.JPAService; +import org.apache.oozie.service.LiteWorkflowStoreService; +import org.apache.oozie.service.SchemaService; +import org.apache.oozie.service.Services; +import org.apache.oozie.test.XDataTestCase; +import org.apache.oozie.util.XConfiguration; + +public class TestWorkflowActionRetryInfoXCommand extends XDataTestCase { + private Services services; + + @Override + protected void setUp() throws Exception { + super.setUp(); + setSystemProperty(SchemaService.WF_CONF_EXT_SCHEMAS, "wf-ext-schema.xsd"); + setSystemProperty(LiteWorkflowStoreService.CONF_USER_RETRY_ERROR_CODE_EXT, ForTestingActionExecutor.TEST_ERROR); + setSystemProperty(Services.CONF_SERVICE_EXT_CLASSES, ExtendedCallableQueueService.class.getName()); + services = new Services(); + services.init(); + services.get(ActionService.class).registerAndInitExecutor(ForTestingActionExecutor.class); + } + + @Override + protected void tearDown() throws Exception { + services.destroy(); + super.tearDown(); + } + + public void testRetryConsoleUrl() throws Exception { + Configuration conf = new XConfiguration(); + File workflowUri = new File(getTestCaseDir(), "workflow.xml"); + + //@formatter:off + String appXml = "<workflow-app xmlns=\"uri:oozie:workflow:0.3\" name=\"wf-fork\">" + + "<start to=\"action1\"/>" + +"<action name=\"action1\" retry-max=\"2\" retry-interval=\"0\">" + + "<test xmlns=\"uri:test\">" + + "<signal-value>${wf:conf('signal-value')}</signal-value>" + + "<external-status>${wf:conf('external-status')}</external-status> " + + "<external-childIds>${wf:conf('external-status')}</external-childIds> " + + "<error>${wf:conf('error')}</error>" + + "<avoid-set-execution-data>${wf:conf('avoid-set-execution-data')}</avoid-set-execution-data>" + + "<avoid-set-end-data>${wf:conf('avoid-set-end-data')}</avoid-set-end-data>" + + "<running-mode>async-error</running-mode>" + + "</test>" + + "<ok to=\"end\"/>" + + "<error to=\"kill\"/>" + + "</action>" + + "<kill name=\"kill\"><message>killed</message></kill>" + + "<end name=\"end\"/>" + + "</workflow-app>"; + //@Formatter:on + writeToFile(appXml, workflowUri); + conf.set(OozieClient.APP_PATH, workflowUri.toURI().toString()); + conf.set(OozieClient.USER_NAME, getTestUser()); + conf.set("external-status", "error"); + conf.set("signal-value", "based_on_action_status"); + conf.set("external-childIds", "1"); + + SubmitXCommand sc = new SubmitXCommand(conf); + final String jobId = sc.call(); + new StartXCommand(jobId).call(); + final WorkflowActionsGetForJobJPAExecutor actionsGetExecutor = new WorkflowActionsGetForJobJPAExecutor(jobId); + final JPAService jpaService = Services.get().get(JPAService.class); + + waitFor(20 * 1000, new Predicate() { + public boolean evaluate() throws Exception { + List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor); + WorkflowActionBean action = null; + for (WorkflowActionBean bean : actions) { + if (bean.getType().equals("test")) { + action = bean; + break; + } + } + return (action != null && action.getUserRetryCount() == 2); + } + }); + + List<WorkflowActionBean> actions = jpaService.execute(actionsGetExecutor); + WorkflowActionBean action = null; + for (WorkflowActionBean bean : actions) { + if (bean.getType().equals("test")) { + action = bean; + break; + } + } + WorkflowActionRetryInfoXCommand command = new WorkflowActionRetryInfoXCommand(action.getId()); + List<Map<String, String>> retriesList = command.call(); + assertEquals(2, retriesList.size()); + assertEquals(2, action.getUserRetryCount()); + + assertEquals(retriesList.get(0).get(JsonTags.ACTION_ATTEMPT), "1"); + assertEquals(retriesList.get(0).get(JsonTags.WORKFLOW_ACTION_START_TIME), + JsonUtils.formatDateRfc822(action.getStartTime())); + + assertNotNull(retriesList.get(0).get(JsonTags.WORKFLOW_ACTION_CONSOLE_URL)); + assertNotNull(retriesList.get(0).get(JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS)); + + assertNotNull(retriesList.get(1).get(JsonTags.WORKFLOW_ACTION_CONSOLE_URL)); + assertNotNull(retriesList.get(1).get(JsonTags.WORKFLOW_ACTION_EXTERNAL_CHILD_IDS)); + assertEquals(retriesList.get(1).get(JsonTags.WORKFLOW_ACTION_END_TIME), + JsonUtils.formatDateRfc822(action.getEndTime())); + + } + +} http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java ---------------------------------------------------------------------- diff --git a/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java b/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java index 60b7735..c76e6d8 100644 --- a/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java +++ b/core/src/test/java/org/apache/oozie/servlet/MockDagEngineService.java @@ -38,7 +38,6 @@ import org.apache.oozie.client.rest.JMSConnectionInfoBean; import org.apache.oozie.client.rest.RestConstants; import org.apache.oozie.service.DagEngineService; import org.apache.oozie.util.XmlUtils; -import org.json.simple.JSONValue; public class MockDagEngineService extends DagEngineService { public static final String JOB_ID = "job-"; @@ -235,6 +234,11 @@ public class MockDagEngineService extends DagEngineService { return (externalId.equals("external-valid")) ? "id-valid" : null; } + public List<Map<String, String>> getWorkflowActionRetries(String actionId) throws DagEngineException { + did = RestConstants.JOB_SHOW_ACTION_RETRIES_PARAM; + return new ArrayList<Map<String, String>>(); + } + private int validateWorkflowIdx(String jobId) throws DagEngineException { int idx = -1; try { http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/core/src/test/resources/wf-ext-schema.xsd ---------------------------------------------------------------------- diff --git a/core/src/test/resources/wf-ext-schema.xsd b/core/src/test/resources/wf-ext-schema.xsd index 48aecf5..3e2ab3e 100644 --- a/core/src/test/resources/wf-ext-schema.xsd +++ b/core/src/test/resources/wf-ext-schema.xsd @@ -26,6 +26,7 @@ <xs:sequence> <xs:element name="signal-value" type="xs:string" minOccurs="1" maxOccurs="unbounded"/> <xs:element name="external-status" type="xs:string" minOccurs="1" maxOccurs="unbounded"/> + <xs:element name="external-childIds" type="xs:string" minOccurs="0" maxOccurs="unbounded"/> <xs:element name="error" type="xs:string" minOccurs="0" maxOccurs="unbounded"/> <xs:element name="avoid-set-execution-data" type="xs:string" minOccurs="0" maxOccurs="unbounded"/> <xs:element name="avoid-set-end-data" type="xs:string" minOccurs="0" maxOccurs="unbounded"/> http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/docs/src/site/twiki/DG_CommandLineTool.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki index 35eee40..2dbbd4c 100644 --- a/docs/src/site/twiki/DG_CommandLineTool.twiki +++ b/docs/src/site/twiki/DG_CommandLineTool.twiki @@ -105,6 +105,7 @@ oozie job <OPTIONS> : job operations -sladisable disables sla alerts for the job and its children -slaenable enables sla alerts for the job and its children -slachange Update sla param for jobs, supported param are should-start, should-end and max-duration + -retries Get information of the retry attempts for a given workflow action. </verbatim> @@ -625,6 +626,34 @@ Job ID Status Started Ended .---------------------------------------------------------------------------------------------------- </verbatim> +---+++ Listing all retry attempts of a workflow action + +When retry-max is specified for an action in the workflow definition, and there is a failure, it will be retried till it succeeds or retry-max attempts are exhausted. To get information on all the retry attempts, =-retries= command can be used. + +<verbatim> +$ oozie job -retries 0000000-161212175234862-oozie-puru-W@pig-node -oozie http://localhost:11000/oozie + +ID : 0000000-161212175234862-oozie-puru-W@pig-node +------------------------------------------------------------------------------------------------------------------------------------ +Attempt : 1 +Start Time : Tue, 13 Dec 2016 01:56:24 GMT +End Time : Tue, 13 Dec 2016 01:56:27 GMT +Console URL : http://localhost:50030/jobdetails.jsp?jobid=job_201612051339_2650 +------------------------------------------------------------------------------------------------------------------------------------ +Attempt : 2 +Start Time : Tue, 13 Dec 2016 01:56:24 GMT +End Time : Tue, 13 Dec 2016 01:56:27 GMT +Console URL : http://localhost:50030/jobdetails.jsp?jobid=job_201612051339_2650 +------------------------------------------------------------------------------------------------------------------------------------ +Attempt : 3 +Start Time : Tue, 13 Dec 2016 01:56:24 GMT +End Time : Tue, 13 Dec 2016 01:56:27 GMT +Console URL : http://localhost:50030/jobdetails.jsp?jobid=job_201612051339_2650 +------------------------------------------------------------------------------------------------------------------------------------ +$ +</verbatim> + + ---+++ Checking the xml definition of a Workflow, Coordinator or Bundle Job Example: http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/docs/src/site/twiki/WebServicesAPI.twiki ---------------------------------------------------------------------- diff --git a/docs/src/site/twiki/WebServicesAPI.twiki b/docs/src/site/twiki/WebServicesAPI.twiki index b76e934..8406da6 100644 --- a/docs/src/site/twiki/WebServicesAPI.twiki +++ b/docs/src/site/twiki/WebServicesAPI.twiki @@ -1353,6 +1353,37 @@ GET /oozie/v1/job/0000001-111219170928042-oozie-joe-C?show=info&filter=status%21 </verbatim> This retrieves coordinator actions except for SUCCEEDED status, which is useful for debugging. +*Retrive information of the retry attempts of the workflow action:* + +<verbatim> +GET oozie/v2/job/0000000-161212175234862-oozie-puru-W@pig-node?show=retries +</verbatim> + +*Response* + +<verbatim> +HTTP/1.1 200 OK +Content-Type: application/json;charset=UTF-8 +. +{ + "retries": + [ + { + "startTime": "Tue, 13 Dec 2016 01:54:13 GMT", + "consoleUrl": "http://localhost:50030/jobdetails.jsp?jobid=job_201612051339_2648";, + "endTime": "Tue, 13 Dec 2016 01:54:20 GMT", + "attempt": "1" + }, + { + "startTime": "Tue, 13 Dec 2016 01:55:20 GMT", + "consoleUrl": "http://localhost:50030/jobdetails.jsp?jobid=job_201612051339_2649";, + "endTime": "Tue, 13 Dec 2016 01:55:24 GMT", + "attempt": "2" + } + ] +} +</verbatim> + ---++++ Job Application Definition A HTTP GET request retrieves the workflow or a coordinator job definition file. http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index 8a4aad0..ca9cd43 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 4.4.0 release (trunk - unreleased) +OOZIE-2691 Show workflow action retry information in UI (puru) OOZIE-2709 Log the substituted pig script by expanding macros (satishsaley) OOZIE-2732 Remove login server example (rkanter via abhishekbafna) OOZIE-2756 Extend HTTPS configuration settings for embedded Jetty (asasvari via abhishekbafna) http://git-wip-us.apache.org/repos/asf/oozie/blob/bf2802e4/webapp/src/main/webapp/oozie-console.js ---------------------------------------------------------------------- diff --git a/webapp/src/main/webapp/oozie-console.js b/webapp/src/main/webapp/oozie-console.js index 11c6940..7b20e91 100644 --- a/webapp/src/main/webapp/oozie-console.js +++ b/webapp/src/main/webapp/oozie-console.js @@ -453,7 +453,8 @@ function jobDetailsPopup(response, request) { var appName = jobDetails["appName"]; var jobActionStatus = new Ext.data.JsonStore({ data: jobDetails["actions"], - fields: ['id', 'name', 'type', 'startTime', 'retries', 'consoleUrl', 'endTime', 'externalId', 'status', 'trackerUri', 'workflowId', 'errorCode', 'errorMessage', 'conf', 'transition', 'externalStatus', 'externalChildIDs'] + fields: ['id', 'name', 'type', 'startTime', 'consoleUrl', 'endTime', 'externalId', 'status', 'userRetryCount' ,'trackerUri', + 'workflowId', 'errorCode', 'errorMessage', 'conf', 'transition', 'externalStatus', 'externalChildIDs'] }); var formFieldSet = new Ext.form.FieldSet({ @@ -623,6 +624,7 @@ function jobDetailsPopup(response, request) { function showActionContextMenu(thisGrid, rowIndex, cellIndex, e) { var actionStatus = thisGrid.store.data.items[rowIndex].data; actionDetailsGridWindow(actionStatus); + function actionDetailsGridWindow(actionStatus) { var formFieldSet = new Ext.form.FieldSet({ title: actionStatus.actionName, @@ -666,6 +668,12 @@ function jobDetailsPopup(response, request) { width: 400, value: actionStatus["status"] }, { + fieldLabel: 'Retries', + editable: false, + width: 400, + name: 'userRetryCount', + value: actionStatus["userRetryCount"] + }, { fieldLabel: 'Error Code', editable: false, name: 'errorCode', @@ -716,66 +724,244 @@ function jobDetailsPopup(response, request) { //width: 540, items: [formFieldSet] }); + var actionInfoPanel = new Ext.TabPanel({ + activeTab: 0, + autoHeight: true, + deferredRender: false, + items: [ { + title: 'Action Info', + items: detail + }, { + title: 'Action Configuration', + items: new Ext.form.TextArea({ + fieldLabel: 'Configuration', + editable: false, + name: 'config', + height: 350, + width: 540, + autoScroll: true, + value: actionStatus["conf"] + }) + + }], + tbar: [{ + text: " ", + icon: 'ext-2.2/resources/images/default/grid/refresh.gif', + handler: function() { + var a = win.items.get(0).getActiveTab(); + if (a.title == "Action retries") { + fetchActionRetries(workflowId + "@" + actionStatus["name"]); + } + else { + refreshActionDetails(workflowId + "@" + actionStatus["name"], detail, urlUnit); + } + } + }] + }); + var actionRetries; var urlUnit = new Ext.FormPanel(); - populateUrlUnit(actionStatus, urlUnit); + populateUrlUnit(actionStatus["consoleUrl"], actionStatus["externalChildIDs"], urlUnit); var win = new Ext.Window({ title: 'Action (Name: ' + actionStatus["name"] + '/JobId: ' + workflowId + ')', closable: true, width: 560, autoHeight: true, plain: true, - items: [new Ext.TabPanel({ - activeTab: 0, - autoHeight: true, - deferredRender: false, - items: [ { - title: 'Action Info', - items: detail - }, { - title: 'Action Configuration', - items: new Ext.form.TextArea({ - fieldLabel: 'Configuration', - editable: false, - name: 'config', - height: 350, - width: 540, - autoScroll: true, - value: actionStatus["conf"] + items: [actionInfoPanel] + }); + + var isLoadedActionRetries = false; + + actionInfoPanel.addListener("tabchange", function(panel, selectedTab) { + if (selectedTab.title == "Action retries") { + if ( !isLoadedActionRetries ) { + fetchActionRetries(workflowId + "@" + actionStatus["name"]); + isLoadedActionRetries = true; + } + } + }); + + var reTriesPanel = new Ext.TabPanel(); + + function fetchActionRetries(actionId) { + Ext.Ajax.request({ + url: getOozieBase() + 'job/' + actionId + "?show=retries", + timeout: 300000, + success: function(response, request) { + actionRetries = JSON.parse(response.responseText); + var currentTabs = []; + reTriesPanel.items.each(function(item) { + currentTabs.push(item); + }); + populateRetries(actionStatus, actionRetries, reTriesPanel); + currentTabs.forEach(function(item) { + reTriesPanel.remove(item); }) - }], - tbar: [{ - text: " ", - icon: 'ext-2.2/resources/images/default/grid/refresh.gif', - handler: function() { - refreshActionDetails(workflowId+"@"+actionStatus["name"], detail, urlUnit); + } + }); + } + populateRetries(actionStatus, actionRetries, reTriesPanel); + + function populateRetries(actionStatus, actionRetries, reTriesPanel) { + var retryCount = getRetryCount(actionStatus, actionRetries); + for (i = 0; i < retryCount; i++) { + var attemptNumber = ' Attempt - ' + (i + 1); + if (actionRetries) { + var retriesJson = actionRetries['retries']; + attemptNumber = ' Attempt - ' + retriesJson[i]["attempt"]; + } + var form = new Ext.FormPanel({ + title: attemptNumber, + width: 500, + height: 400, + style: 'background-color: #eaf1fb' + }); + if (actionRetries) { + var retriesJson = actionRetries['retries']; + addTextField(form, "Start Time", "startTime", retriesJson[i]); + addTextField(form, "End Time", "endTime", retriesJson[i]); + populateRetriesUrl(form, "Console URL", "consoleUrl", retriesJson[i]); + populateRetriesChildUrl(form, retriesJson[i]); + } + reTriesPanel.add(form); + form.show(); + } + reTriesPanel.doLayout(); + } + + function getRetryCount(actionStatus, actionRetries) { + if (actionRetries) { + if (actionRetries['retries']) { + return actionRetries['retries'].length; + } + } else { + return actionStatus["userRetryCount"]; + } + } + function populateRetriesUrl(form, label, key, retriesObj) { + if ( retriesObj[key] ) { + var textUrl = new Ext.form.TriggerField({ + fieldLabel: label, + editable: false, + width: 400, + height: 100, + value: retriesObj[key], + triggerClass: 'x-form-search-trigger', + onTriggerClick: function() { + window.open(retriesObj[text]); } - }] - })] - }); + }); + form.add(textUrl); + } + else { + addTextField(form, label, key, retriesObj); + } + } + + function addTextField(form, label, key, retriesObj) { + if (retriesObj[key]) { + var textUrl = new Ext.form.TextField({ + fieldLabel: label, + width: 400, + value: retriesObj[key] + }); + form.add(textUrl); + } else { + var textUrl = new Ext.form.TextField({ + fieldLabel: label, + width: 400, + value: 'n/a' + }); + form.add(textUrl); + } + } + + function populateRetriesChildUrl(form, retriesObj) { + populateUrlUnit(retriesObj["consoleUrl"], retriesObj["externalChildIDs"], form); + } // Tab to show list of child Job URLs var childJobsItem = { - title : 'Child Job URLs', - autoScroll : true, - frame : true, - labelAlign : 'right', - labelWidth : 70, + title: 'Child Job URLs', + autoScroll: true, + frame: true, + labelAlign: 'right', + labelWidth: 70, height: 350, width: 540, - items : urlUnit + items: urlUnit }; - if (actionStatus.type == "pig" || actionStatus.type == "hive" || actionStatus.type == "map-reduce" - || actionStatus.type == "hive2" || actionStatus.type == "sqoop" || actionStatus.type == "distcp" - || actionStatus.type == "spark") { + if (actionStatus.type == "pig" || actionStatus.type == "hive" || actionStatus.type == "map-reduce" || + actionStatus.type == "hive2" || actionStatus.type == "sqoop" || actionStatus.type == "distcp" || + actionStatus.type == "spark") { var tabPanel = win.items.get(0); tabPanel.add(childJobsItem); } + // Tab to show list of Retries + var retriesActionItem = { + title: 'Action retries', + autoScroll: true, + frame: true, + labelAlign: 'right', + labelWidth: 70, + height: 350, + width: 540, + items: reTriesPanel + }; + if (actionStatus["userRetryCount"] > 0) { + var tabPanel = win.items.get(0); + tabPanel.add(retriesActionItem); + } win.setPosition(50, 50); win.show(); } } + function populateUrlUnit(consoleUrl, externalChildIDs, urlUnit) { + if (consoleUrl && externalChildIDs && externalChildIDs != "null") { + var urlPrefix = consoleUrl.trim().split(/_/)[0]; + // externalChildIds is a comma-separated string of each child job + // ID. + // Create URL list by appending jobID portion after stripping "job" + var jobIds = externalChildIDs.split(/,/); + var count = 1; + jobIds.forEach(function(jobId) { + jobId = jobId.trim().split(/job/); + if (jobId.length > 1) { + jobId = jobId[1]; + var jobUrl = new Ext.form.TriggerField({ + fieldLabel : 'Child Job ' + count, + editable : false, + name : 'childJobURLs', + width : 400, + value : urlPrefix + jobId, + triggerClass : 'x-form-search-trigger', + onTriggerClick : function() { + window.open(urlPrefix + jobId); + } + }); + if (jobId != undefined) { + urlUnit.add(jobUrl); + count++; + } + } + }); + if (count == 1) { + var note = new Ext.form.TextField({ + fieldLabel : 'Child Job', + value : 'n/a' + }); + urlUnit.add(note); + } + } else { + var note = new Ext.form.TextField({ + fieldLabel : 'Child Job', + value : 'n/a' + }); + urlUnit.add(note); + } + } - function populateUrlUnit(actionStatus, urlUnit) { + function populateRetries(actionStatus, urlUnit) { var consoleUrl = actionStatus["consoleUrl"]; var externalChildIDs = actionStatus["externalChildIDs"]; if (consoleUrl && externalChildIDs && externalChildIDs != "null") { @@ -830,7 +1016,7 @@ function jobDetailsPopup(response, request) { var results = JSON.parse(response.responseText); detail.getForm().setValues(results); urlUnit.getForm().setValues(results); - populateUrlUnit(results, urlUnit); + populateUrlUnit(results["consoleUrl"], results["externalChildIDs"], urlUnit); } }); } @@ -1354,6 +1540,12 @@ function coordJobDetailsPopup(response, request) { width: 400, value: actionStatus["status"] }, { + fieldLabel: 'Retries', + editable: false, + width: 400, + name: 'userRetryCount', + value: actionStatus["userRetryCount"] + }, { fieldLabel: 'Error Code', editable: false, name: 'errorCode',
