This is an automated email from the ASF dual-hosted git repository. asalamon74 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/oozie.git
The following commit(s) were added to refs/heads/master by this push: new a0b5965 OOZIE-3066 Possibility to set retry-max globally (qsbao via asalamon74) a0b5965 is described below commit a0b59658d90d4a83d5101c2f62dbad697fb5d45b Author: Andras Salamon <asalamo...@apache.org> AuthorDate: Thu Jan 9 12:14:53 2020 +0100 OOZIE-3066 Possibility to set retry-max globally (qsbao via asalamon74) --- core/src/main/java/org/apache/oozie/ErrorCode.java | 1 + .../oozie/service/LiteWorkflowStoreService.java | 12 +++++- core/src/main/resources/oozie-default.xml | 8 ++++ .../service/TestLiteWorkflowStoreService.java | 49 ++++++++++++++++++++++ docs/src/site/markdown/WorkflowFunctionalSpec.md | 7 +++- release-log.txt | 1 + 6 files changed, 74 insertions(+), 4 deletions(-) diff --git a/core/src/main/java/org/apache/oozie/ErrorCode.java b/core/src/main/java/org/apache/oozie/ErrorCode.java index 8e98535..8f4e21d 100644 --- a/core/src/main/java/org/apache/oozie/ErrorCode.java +++ b/core/src/main/java/org/apache/oozie/ErrorCode.java @@ -182,6 +182,7 @@ public enum ErrorCode { E0820(XLog.STD, "Action user retry max [{0}] is over system defined max [{1}], re-assign to use system max."), E0821(XLog.STD, "Received early callback for action still in PREP state; will wait [{0}]ms and requeue up to [{1}] more times"), E0822(XLog.STD, "Received early callback for action [{0}] while still in PREP state and exhausted all requeues"), + E0823(XLog.STD, "System defined default user retry [{0}] is over system defined max [{1}], re-assign to use system max."), E0900(XLog.OPS, "JobTracker [{0}] not allowed, not in Oozie''s whitelist. Allowed values are: {1}"), E0901(XLog.OPS, "NameNode [{0}] not allowed, not in Oozie''s whitelist. Allowed values are: {1}"), diff --git a/core/src/main/java/org/apache/oozie/service/LiteWorkflowStoreService.java b/core/src/main/java/org/apache/oozie/service/LiteWorkflowStoreService.java index 235f634..636fd5c 100644 --- a/core/src/main/java/org/apache/oozie/service/LiteWorkflowStoreService.java +++ b/core/src/main/java/org/apache/oozie/service/LiteWorkflowStoreService.java @@ -53,11 +53,14 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import com.google.common.annotations.VisibleForTesting; + public abstract class LiteWorkflowStoreService extends WorkflowStoreService { public static final String CONF_PREFIX = Service.CONF_PREFIX + "LiteWorkflowStoreService."; public static final String CONF_PREFIX_USER_RETRY = CONF_PREFIX + "user.retry."; public static final String CONF_USER_RETRY_MAX = CONF_PREFIX_USER_RETRY + "max"; + public static final String CONF_USER_RETRY_DEFAULT = CONF_PREFIX_USER_RETRY + "default"; public static final String CONF_USER_RETRY_INTEVAL = CONF_PREFIX_USER_RETRY + "inteval"; public static final String CONF_USER_RETRY_POLICY = CONF_PREFIX_USER_RETRY + "policy"; public static final String CONF_USER_RETRY_ERROR_CODE = CONF_PREFIX_USER_RETRY + "error.code"; @@ -155,7 +158,8 @@ public abstract class LiteWorkflowStoreService extends WorkflowStoreService { return ret; } - private static int getUserRetryMax(NodeHandler.Context context) throws WorkflowException { + @VisibleForTesting + protected static int getUserRetryMax(NodeHandler.Context context) throws WorkflowException { XLog log = XLog.getLog(LiteWorkflowStoreService.class); int ret = ConfigurationService.getInt(CONF_USER_RETRY_MAX); int max = ret; @@ -174,7 +178,11 @@ public abstract class LiteWorkflowStoreService extends WorkflowStoreService { } } else { - ret = 0; + ret = ConfigurationService.getInt(CONF_USER_RETRY_DEFAULT); + if (ret > max) { + log.warn(ErrorCode.E0823.getTemplate(), ret, max); + ret = max; + } } return ret; } diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml index 13d7f1e..ab7b8d3 100644 --- a/core/src/main/resources/oozie-default.xml +++ b/core/src/main/resources/oozie-default.xml @@ -2293,6 +2293,14 @@ will be the requeue interval for the actions which are waiting for a long time w </property> <property> + <name>oozie.service.LiteWorkflowStoreService.user.retry.default</name> + <value>0</value> + <description> + Default automatic retry count for workflow action. + </description> + </property> + + <property> <name>oozie.service.LiteWorkflowStoreService.user.retry.inteval</name> <value>10</value> <description> diff --git a/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowStoreService.java b/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowStoreService.java index 06473d0..043dcbc 100644 --- a/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowStoreService.java +++ b/core/src/test/java/org/apache/oozie/service/TestLiteWorkflowStoreService.java @@ -25,6 +25,10 @@ import org.apache.oozie.ForTestingActionExecutor; import org.apache.oozie.service.Services; import org.apache.oozie.service.WorkflowStoreService; import org.apache.oozie.test.XTestCase; +import org.apache.oozie.workflow.WorkflowException; +import org.apache.oozie.workflow.lite.NodeDef; +import org.apache.oozie.workflow.lite.NodeHandler; +import org.mockito.Mockito; public class TestLiteWorkflowStoreService extends XTestCase { @@ -82,5 +86,50 @@ public class TestLiteWorkflowStoreService extends XTestCase { assertTrue(allowedRetryCodes.contains(LiteWorkflowStoreService.USER_ERROR_CODE_ALL)); } + public void testGetUserRetryMax() throws WorkflowException { + NodeHandler.Context context = Mockito.mock(NodeHandler.Context.class); + NodeDef nodeDef = Mockito.mock(NodeDef.class); + Mockito.when(context.getNodeDef()).thenReturn(nodeDef); + + assertEquals("System defined user retry max should be 3", + 3, ConfigurationService.getInt("oozie.service.LiteWorkflowStoreService.user.retry.max")); + assertEquals("System defined user retry default should be 0", + 0, ConfigurationService.getInt("oozie.service.LiteWorkflowStoreService.user.retry.default")); + + // User defined retry-max not greater than system max + Mockito.when(nodeDef.getUserRetryMax()).thenReturn("2"); + assertEquals("Should return user defined if user defined is not over system max", + 2, LiteWorkflowStoreService.getUserRetryMax(context)); + + // User defined retry-max greater than system max + ConfigurationService.set("oozie.service.LiteWorkflowStoreService.user.retry.max", "1"); + ConfigurationService.set("oozie.service.LiteWorkflowStoreService.user.retry.default", "0"); + Mockito.when(nodeDef.getUserRetryMax()).thenReturn("2"); + assertEquals("Should return system max if user defined is over system max", + 1, LiteWorkflowStoreService.getUserRetryMax(context)); + + // User defined retry-max is an invalid number + Mockito.when(nodeDef.getUserRetryMax()).thenReturn("abc"); + try { + LiteWorkflowStoreService.getUserRetryMax(context); + fail("Should have thrown exception because user defined retry-max is an invalid number"); + } catch (WorkflowException ex) { + // Pass + } + + // User defined retry-max is null, return system default + ConfigurationService.set("oozie.service.LiteWorkflowStoreService.user.retry.max", "3"); + ConfigurationService.set("oozie.service.LiteWorkflowStoreService.user.retry.default", "2"); + Mockito.when(nodeDef.getUserRetryMax()).thenReturn("null"); + assertEquals("Should return system default if user defined is null", + 2, LiteWorkflowStoreService.getUserRetryMax(context)); + + // User defined retry-max is null, and system default greater than system max, return system max + ConfigurationService.set("oozie.service.LiteWorkflowStoreService.user.retry.max", "1"); + ConfigurationService.set("oozie.service.LiteWorkflowStoreService.user.retry.default", "2"); + Mockito.when(nodeDef.getUserRetryMax()).thenReturn("null"); + assertEquals("Should return system max if user defined is null and system default is over system max", + 1, LiteWorkflowStoreService.getUserRetryMax(context)); + } } diff --git a/docs/src/site/markdown/WorkflowFunctionalSpec.md b/docs/src/site/markdown/WorkflowFunctionalSpec.md index 0eae0b5..136f1d7 100644 --- a/docs/src/site/markdown/WorkflowFunctionalSpec.md +++ b/docs/src/site/markdown/WorkflowFunctionalSpec.md @@ -2700,13 +2700,16 @@ User-Retry allows user to give certain number of reties (must not exceed system that problem and fix them when action is in `USER_RETRY` state. If failure or error does not go away after max retries, the action becomes `FAILED` or `ERROR` and Oozie marks workflow job to `FAILED` or `KILLED`. +Oozie administrator can define system default retries by adding this configuration +`oozie.service.LiteWorkflowStoreService.user.retry.default` to `oozie-site.xml`. + Oozie administrator can allow more error codes to be handled for User-Retry. By adding this configuration -`oozie.service.LiteWorkflowStoreService.user.retry.error.code.ext` to `oozie.site.xml` +`oozie.service.LiteWorkflowStoreService.user.retry.error.code.ext` to `oozie-site.xml` and error codes as value, these error codes will be considered as User-Retry after system restart. Since Oozie 4.3, User-retry allows user to mention retry policy. The value for policy can be `periodic` or `exponential`, `periodic` being the default. Oozie administrator can define user retry policy for all workflow -actions by adding this configuration `oozie.service.LiteWorkflowStoreService.user.retry.policy` to `oozie.site.xml`. +actions by adding this configuration `oozie.service.LiteWorkflowStoreService.user.retry.policy` to `oozie-site.xml`. This value will be considered as user retry policy after system restart. This value can be overridden while defining actions in workflow xml if needed. The `retry-interval` should be specified in minutes. diff --git a/release-log.txt b/release-log.txt index 5877766..5acd705 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.3.0 release (trunk - unreleased) +OOZIE-3066 Possibility to set retry-max globally (qsbao via asalamon74) OOZIE-3578 MapReduce counters cannot be used over 120 (dionusos via pbacsko, gezapeti) OOZIE-3573 Fix oozie-default.xml descriptions (arvaiadam via asalamon74) OOZIE-3571 Wrong log string substitution in CredentialsProviderFactory class (arvaiadam via asalamon74)