STORM-166 making min.replication.count and max.wait.time topology configuration.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/01eee7aa Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/01eee7aa Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/01eee7aa Branch: refs/heads/0.11.x-branch Commit: 01eee7aa519c9d63afc38f2f61a53f48913c0a28 Parents: 27a381a Author: Parth Brahmbhatt <[email protected]> Authored: Mon Feb 2 23:37:17 2015 -0800 Committer: Parth Brahmbhatt <[email protected]> Committed: Mon Feb 2 23:37:17 2015 -0800 ---------------------------------------------------------------------- conf/defaults.yaml | 4 ++-- docs/documentation/nimbus-ha-design.md | 10 +++++----- storm-core/src/clj/backtype/storm/daemon/nimbus.clj | 6 +++--- storm-core/src/jvm/backtype/storm/Config.java | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/storm/blob/01eee7aa/conf/defaults.yaml ---------------------------------------------------------------------- diff --git a/conf/defaults.yaml b/conf/defaults.yaml index 7fea11c..fbea948 100644 --- a/conf/defaults.yaml +++ b/conf/defaults.yaml @@ -65,8 +65,8 @@ nimbus.task.launch.secs: 120 nimbus.reassign: true nimbus.file.copy.expiration.secs: 600 nimbus.topology.validator: "backtype.storm.nimbus.DefaultTopologyValidator" -nimbus.min.replication.count: 0 -nimbus.max.replication.wait.time.sec: 0 +topology.min.replication.count: 1 +topology.max.replication.wait.time.sec: 60 nimbus.credential.renewers.freq.secs: 600 ### ui.* configs are for the master http://git-wip-us.apache.org/repos/asf/storm/blob/01eee7aa/docs/documentation/nimbus-ha-design.md ---------------------------------------------------------------------- diff --git a/docs/documentation/nimbus-ha-design.md b/docs/documentation/nimbus-ha-design.md index 9c38851..00fd115 100644 --- a/docs/documentation/nimbus-ha-design.md +++ b/docs/documentation/nimbus-ha-design.md @@ -209,12 +209,12 @@ downloading the code-distrbutor meta file it contacts zookeeper in order to figu actual code/config and to get the current replication count. An alternative is to use "org.apache.storm.hdfs.ha.codedistributor.HDFSCodeDistributor" which relies on HDFS but does not add extra load on zookeeper and will make topology submission faster. -* nimbus.min.replication.count : Minimum number of nimbus hosts where the code must be replicated before leader nimbus -can mark the topology as active and create assignments. Default is 0. in case of HDFSCodeDistributor this represents number -of data nodes insted of nimbus hosts where code must be replicated before activating topology. -* nimbus.max.replication.wait.time.sec: Maximum wait time for the nimbus host replication to achieve the nimbus.min.replication.count. +* topology.min.replication.count : Minimum number of nimbus hosts where the code must be replicated before leader nimbus +can mark the topology as active and create assignments. Default is 1. in case of HDFSCodeDistributor this represents number +of data nodes instead of nimbus hosts where code must be replicated before activating topology. +* topology.max.replication.wait.time.sec: Maximum wait time for the nimbus host replication to achieve the nimbus.min.replication.count. Once this time is elapsed nimbus will go ahead and perform topology activation tasks even if required nimbus.min.replication.count is not achieved. -The default is 0 seconds, a value of -1 indicates to wait for ever. +The default is 60 seconds, a value of -1 indicates to wait for ever. *nimbus.code.sync.freq.secs: frequency at which the background thread which syncs code for locally missing topologies will run. default is 5 minutes. Note: Even though all nimbus hosts have watchers on zookeeper to be notified immediately as soon as a new topology is available for code http://git-wip-us.apache.org/repos/asf/storm/blob/01eee7aa/storm-core/src/clj/backtype/storm/daemon/nimbus.clj ---------------------------------------------------------------------- diff --git a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj index e07e6d1..2528825 100644 --- a/storm-core/src/clj/backtype/storm/daemon/nimbus.clj +++ b/storm-core/src/clj/backtype/storm/daemon/nimbus.clj @@ -340,8 +340,8 @@ )) (defn- wait-for-desired-code-replication [nimbus conf storm-id] - (let [min-replication-count (conf NIMBUS-MIN-REPLICATION-COUNT) - max-replication-wait-time (conf NIMBUS-MAX-REPLICATION-WAIT-TIME-SEC) + (let [min-replication-count (conf TOPOLOGY-MIN-REPLICATION-COUNT) + max-replication-wait-time (conf TOPOLOGY-MAX-REPLICATION-WAIT-TIME-SEC) total-wait-time (atom 0) current-replication-count (atom (if (:code-distributor nimbus) (.getReplicationCount (:code-distributor nimbus) storm-id) 0))] (if (:code-distributor nimbus) @@ -1129,7 +1129,7 @@ (.set-credentials! storm-cluster-state storm-id credentials storm-conf) (setup-storm-code nimbus conf storm-id uploadedJarLocation storm-conf topology) (.setup-code-distributor! storm-cluster-state storm-id (:nimbus-host-port-info nimbus)) - (wait-for-desired-code-replication nimbus conf storm-id) + (wait-for-desired-code-replication nimbus total-storm-conf storm-id) (.setup-heartbeats! storm-cluster-state storm-id) (let [thrift-status->kw-status {TopologyInitialStatus/INACTIVE :inactive TopologyInitialStatus/ACTIVE :active}] http://git-wip-us.apache.org/repos/asf/storm/blob/01eee7aa/storm-core/src/jvm/backtype/storm/Config.java ---------------------------------------------------------------------- diff --git a/storm-core/src/jvm/backtype/storm/Config.java b/storm-core/src/jvm/backtype/storm/Config.java index df330a0..c0668ee 100644 --- a/storm-core/src/jvm/backtype/storm/Config.java +++ b/storm-core/src/jvm/backtype/storm/Config.java @@ -1252,8 +1252,8 @@ public class Config extends HashMap<String, Object> { * is allowed to perform topology activation tasks like setting up heartbeats/assignments * and marking the topology as active. default is 0. */ - public static final String NIMBUS_MIN_REPLICATION_COUNT = "nimbus.min.replication.count"; - public static final Object NIMBUS_MIN_REPLICATION_COUNT_SCHEMA = Number.class; + public static final String TOPOLOGY_MIN_REPLICATION_COUNT = "topology.min.replication.count"; + public static final Object TOPOLOGY_MIN_REPLICATION_COUNT_SCHEMA = Number.class; /** * Maximum wait time for the nimbus host replication to achieve the nimbus.min.replication.count. @@ -1261,8 +1261,8 @@ public class Config extends HashMap<String, Object> { * if required nimbus.min.replication.count is not achieved. The default is 0 seconds, a value of * -1 indicates to wait for ever. */ - public static final String NIMBUS_MAX_REPLICATION_WAIT_TIME_SEC = "nimbus.max.replication.wait.time.sec"; - public static final Object NIMBUS_MAX_REPLICATION_WAIT_TIME_SEC_SCHEMA = Number.class; + public static final String TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC = "nimbus.max.replication.wait.time.sec"; + public static final Object TOPOLOGY_MAX_REPLICATION_WAIT_TIME_SEC_SCHEMA = Number.class; /** * How often nimbus's background thread to sync code for missing topologies should run.
