Repository: incubator-myriad Updated Branches: refs/heads/constraints [created] f90a0407c refs/heads/docker-image [created] 7db44a4d7 refs/heads/executor-only-application [created] b5c64ffff refs/heads/issue_14 [created] 359a2641f refs/heads/issue_16 [created] c7d4d13ca refs/heads/master [created] 0c2aeae8b refs/heads/multi-project [created] 82ebfdb06 refs/heads/myriadha [created] f90a0407c refs/heads/phase1 [created] 1604aeced
Made reconciliation params configurable Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/c7d4d13c Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/c7d4d13c Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/c7d4d13c Branch: refs/heads/issue_16 Commit: c7d4d13ca16cbd3a5aaaafc06e329ce0cd7432c7 Parents: a55f837 Author: Mohit Soni <mohitsoni1...@gmail.com> Authored: Wed Mar 4 19:16:04 2015 +0000 Committer: Mohit Soni <mohitsoni1...@gmail.com> Committed: Wed Mar 4 19:16:04 2015 +0000 ---------------------------------------------------------------------- docs/myriad-configuration.md | 6 ++ .../configuration/MyriadConfiguration.java | 63 ++++++++++++++------ .../ebay/myriad/scheduler/ReconcileService.java | 9 ++- 3 files changed, 56 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/c7d4d13c/docs/myriad-configuration.md ---------------------------------------------------------------------- diff --git a/docs/myriad-configuration.md b/docs/myriad-configuration.md index d2bdd91..471bc2d 100644 --- a/docs/myriad-configuration.md +++ b/docs/myriad-configuration.md @@ -32,6 +32,12 @@ zkServers: localhost:2181 # ZK Session timeout zkTimeout: 20000 +# The delay between reconcile attempts +reconciliationDelayMillis: 10000 + +# Maximum number of reconciliation attempts. +maxReconcileAttempts: 10 + # The node manager profiles. The REST API to flex up expects one of the profiles defined here. # Admin can define custom profiles (requires restart of Resource Manager) profiles: http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/c7d4d13c/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java ---------------------------------------------------------------------- diff --git a/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java b/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java index d31e89a..f3dae79 100644 --- a/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java +++ b/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java @@ -15,6 +15,7 @@ */ package com.ebay.myriad.configuration; +import com.ebay.myriad.scheduler.yarn.interceptor.InterceptorRegistry; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Strings; import org.hibernate.validator.constraints.NotEmpty; @@ -22,7 +23,8 @@ import org.hibernate.validator.constraints.NotEmpty; import java.util.Map; /** - * Myriad Configuration commonly defined in the YML file + * Myriad Configuration commonly defined in the YML file: + * <pre> * mesosMaster: 10.0.2.15:5050 * checkpoint: false * frameworkFailoverTimeout: 43200000 @@ -30,27 +32,30 @@ import java.util.Map; * nativeLibrary: /usr/local/lib/libmesos.so * zkServers: localhost:2181 * zkTimeout: 20000 + * reconcilationDelay: 10000 + * maxReconcileAttempts: 10 * profiles: - * small: - * cpu: 1 - * mem: 1100 - * medium: - * cpu: 2 - * mem: 2048 - * large: - * cpu: 4 - * mem: 4096 + * small: + * cpu: 1 + * mem: 1100 + * medium: + * cpu: 2 + * mem: 2048 + * large: + * cpu: 4 + * mem: 4096 * rebalancer: false * nodemanager: - * jvmMaxMemoryMB: 1024 - * user: hduser - * cpus: 0.2 - * cgroups: false + * jvmMaxMemoryMB: 1024 + * user: hduser + * cpus: 0.2 + * cgroups: false * executor: - * jvmMaxMemoryMB: 256 - * path: file://localhost/usr/local/libexec/mesos/myriad-executor-0.0.1.jar + * jvmMaxMemoryMB: 256 + * path: file://localhost/usr/local/libexec/mesos/myriad-executor-0.0.1.jar * yarnEnvironment: - * YARN_HOME: /usr/local/hadoop + * YARN_HOME: /usr/local/hadoop + * </pre> */ public class MyriadConfiguration { /** @@ -74,6 +79,16 @@ public class MyriadConfiguration { public static final Integer DEFAULT_ZK_TIMEOUT = 20000; + /** + * Default delay between reconcile attempts. + */ + public static final Integer DEFAULT_RECONCILIATION_DELAY_MS = 10000; + + /** + * Maximum number of reconcilation attempts. + */ + public static final Integer DEFAULT_MAX_RECONCILE_ATTEMPTS = 10; + @JsonProperty @NotEmpty private String mesosMaster; @@ -115,6 +130,12 @@ public class MyriadConfiguration { private Integer zkTimeout; @JsonProperty + private Integer reconciliationDelayMillis; + + @JsonProperty + private Integer maxReconcileAttempts; + + @JsonProperty @NotEmpty private Map<String, String> yarnEnvironment; @@ -175,4 +196,12 @@ public class MyriadConfiguration { public Map<String, String> getYarnEnvironment() { return yarnEnvironment; } + + public Integer getReconciliationDelayMillis() { + return this.reconciliationDelayMillis != null ? this.reconciliationDelayMillis : DEFAULT_RECONCILIATION_DELAY_MS; + } + + public Integer getMaxReconcileAttempts() { + return this.maxReconcileAttempts != null ? this.maxReconcileAttempts : DEFAULT_MAX_RECONCILE_ATTEMPTS; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/c7d4d13c/src/main/java/com/ebay/myriad/scheduler/ReconcileService.java ---------------------------------------------------------------------- diff --git a/src/main/java/com/ebay/myriad/scheduler/ReconcileService.java b/src/main/java/com/ebay/myriad/scheduler/ReconcileService.java index 5a3b026..333d971 100644 --- a/src/main/java/com/ebay/myriad/scheduler/ReconcileService.java +++ b/src/main/java/com/ebay/myriad/scheduler/ReconcileService.java @@ -18,9 +18,6 @@ import java.util.Date; public class ReconcileService { private static final Logger LOGGER = LoggerFactory.getLogger(ReconcileService.class); - public static final long DEFAULT_RECONCILATION_DELAY_MS = 10000; - public static final long MAX_RECONCILE_ATTEMPTS = 10; - private SchedulerState state; private MyriadConfiguration cfg; private Date lastReconcileTime; @@ -45,10 +42,12 @@ public class ReconcileService { int attempt = 1; - while (attempt <= MAX_RECONCILE_ATTEMPTS) { + Integer maxReconcileAttempts = this.cfg.getMaxReconcileAttempts(); + Integer reconciliationDelayMillis = this.cfg.getReconciliationDelayMillis(); + while (attempt <= maxReconcileAttempts) { try { // TODO(mohit): Using exponential backoff here, maybe backoff strategy should be configurable. - Thread.sleep(DEFAULT_RECONCILATION_DELAY_MS * attempt); + Thread.sleep(reconciliationDelayMillis * attempt); } catch (InterruptedException e) { LOGGER.error("Interrupted", e); }
