Updated Branches: refs/heads/flume-1.5 e1dbe0eb1 -> 7fc23d7d4
FLUME-2229. Backoff period gets reset too often in OrderSelector (Hari Shreedharan via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/flume/repo Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/7fc23d7d Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/7fc23d7d Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/7fc23d7d Branch: refs/heads/flume-1.5 Commit: 7fc23d7d41757ce75058cead963b5bd54c395727 Parents: e1dbe0e Author: Jarek Jarcec Cecho <[email protected]> Authored: Thu Oct 31 12:54:10 2013 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Thu Oct 31 12:55:22 2013 -0700 ---------------------------------------------------------------------- .../java/org/apache/flume/util/OrderSelector.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flume/blob/7fc23d7d/flume-ng-sdk/src/main/java/org/apache/flume/util/OrderSelector.java ---------------------------------------------------------------------- diff --git a/flume-ng-sdk/src/main/java/org/apache/flume/util/OrderSelector.java b/flume-ng-sdk/src/main/java/org/apache/flume/util/OrderSelector.java index e869930..fd9e81f 100644 --- a/flume-ng-sdk/src/main/java/org/apache/flume/util/OrderSelector.java +++ b/flume-ng-sdk/src/main/java/org/apache/flume/util/OrderSelector.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; /** * A basic implementation of an order selector that implements a simple @@ -39,7 +40,8 @@ import java.util.Map; public abstract class OrderSelector<T> { private static final int EXP_BACKOFF_COUNTER_LIMIT = 16; - private static final long CONSIDER_SEQUENTIAL_RANGE = 2000l; + private static final long CONSIDER_SEQUENTIAL_RANGE = TimeUnit.HOURS + .toMillis(1); private static final long MAX_TIMEOUT = 30000l; private final Map<T, FailureState> stateMap = new LinkedHashMap<T, FailureState>(); @@ -92,12 +94,14 @@ public abstract class OrderSelector<T> { long now = System.currentTimeMillis(); long delta = now - state.lastFail; - //Should we consider this as a new failure? If the failure happened - //within backoff length + a grace period (failed within - //grace period after the component started up again, don't consider this - //a new sequential failure - the component might have failed again while - //trying to recover. If the failure is outside backedoff time + grace period - //consider it a new failure and increase the backoff length. + /* + * When do we increase the backoff period? + * We basically calculate the time difference between the last failure + * and the current one. If this failure happened within one hour of the + * last backoff period getting over, then we increase the timeout, + * since the object did not recover yet. Else we assume this is a fresh + * failure and reset the count. + */ long lastBackoffLength = Math.min(maxTimeout, 1000 * (1 << state.sequentialFails)); long allowableDiff = lastBackoffLength + CONSIDER_SEQUENTIAL_RANGE; if (allowableDiff > delta) {
