Repository: ambari Updated Branches: refs/heads/branch-metrics-dev dd066b945 -> 063335855
AMBARI-5707. Aggregator exceptions due to misconfiguration. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/06333585 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/06333585 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/06333585 Branch: refs/heads/branch-metrics-dev Commit: 0633358555468d2c8c5d6d8d30902a82a5036247 Parents: dd066b9 Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Fri Nov 21 10:15:58 2014 -0800 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Fri Nov 21 10:15:58 2014 -0800 ---------------------------------------------------------------------- .../timeline/AbstractTimelineAggregator.java | 19 ++++++++++--------- .../timeline/TimelineMetricAggregatorHourly.java | 11 ++++++----- .../timeline/TimelineMetricAggregatorMinute.java | 10 ++++++---- .../TimelineMetricClusterAggregator.java | 19 +++++++++++-------- .../TimelineMetricClusterAggregatorHourly.java | 19 ++++++++++--------- .../timeline/TimelineMetricConfiguration.java | 3 ++- 6 files changed, 45 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java index 6bd6507..43ec648 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java @@ -33,6 +33,7 @@ import java.io.File; import java.io.IOException; import java.util.Date; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics .timeline.TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics @@ -42,7 +43,7 @@ public abstract class AbstractTimelineAggregator implements Runnable { protected final PhoenixHBaseAccessor hBaseAccessor; private final Log LOG; private static final ObjectMapper mapper; - protected final long checkpointDelay; + protected final long checkpointDelayMillis; protected final Integer resultsetFetchSize; protected Configuration metricsConf; @@ -54,8 +55,8 @@ public abstract class AbstractTimelineAggregator implements Runnable { Configuration metricsConf) { this.hBaseAccessor = hBaseAccessor; this.metricsConf = metricsConf; - this.checkpointDelay = metricsConf.getInt(AGGREGATOR_CHECKPOINT_DELAY, - 120000); + this.checkpointDelayMillis = SECONDS.toMillis( + metricsConf.getInt(AGGREGATOR_CHECKPOINT_DELAY, 120)); this.resultsetFetchSize = metricsConf.getInt(RESULTSET_FETCH_SIZE, 2000); this.LOG = LogFactory.getLog(this.getClass()); } @@ -63,7 +64,7 @@ public abstract class AbstractTimelineAggregator implements Runnable { @Override public void run() { LOG.info("Started Timeline aggregator thread @ " + new Date()); - Long SLEEP_INTERVAL = getSleepInterval(); + Long SLEEP_INTERVAL = getSleepIntervalMillis(); while (true) { long currentTime = System.currentTimeMillis(); @@ -80,7 +81,7 @@ public abstract class AbstractTimelineAggregator implements Runnable { // Assuming first run, save checkpoint and sleep. // Set checkpoint to 2 minutes in the past to allow the // agents/collectors to catch up - saveCheckPoint(currentTime - checkpointDelay); + saveCheckPoint(currentTime - checkpointDelayMillis); } } catch (IOException io) { LOG.warn("Unable to write last checkpoint time. Resuming sleep.", io); @@ -131,7 +132,7 @@ public abstract class AbstractTimelineAggregator implements Runnable { private boolean isLastCheckPointTooOld(long checkpoint) { return checkpoint != -1 && ((System.currentTimeMillis() - checkpoint) > - getCheckpointCutOffInterval()); + getCheckpointCutOffIntervalMillis()); } private long readCheckPoint() { @@ -164,12 +165,12 @@ public abstract class AbstractTimelineAggregator implements Runnable { // TODO: Abstract out doWork implementation for cluster and host levels protected abstract boolean doWork(long startTime, long endTime); - protected abstract Long getSleepInterval(); + protected abstract Long getSleepIntervalMillis(); protected abstract Integer getCheckpointCutOffMultiplier(); - protected Long getCheckpointCutOffInterval() { - return getCheckpointCutOffMultiplier() * getSleepInterval(); + protected Long getCheckpointCutOffIntervalMillis() { + return getCheckpointCutOffMultiplier() * getSleepIntervalMillis(); } protected abstract boolean isDisabled(); http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java index 6a84024..16f5ab9 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorHourly.java @@ -34,6 +34,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics .timeline.PhoenixTransactSQL.*; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics @@ -55,7 +56,7 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator { private static final String MINUTE_AGGREGATE_HOURLY_CHECKPOINT_FILE = "timeline-metrics-host-aggregator-hourly-checkpoint"; private final String checkpointLocation; - private final Long sleepInterval; + private final Long sleepIntervalMillis; private final Integer checkpointCutOffMultiplier; public TimelineMetricAggregatorHourly(PhoenixHBaseAccessor hBaseAccessor, @@ -69,8 +70,8 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator { checkpointLocation = FilenameUtils.concat(checkpointDir, MINUTE_AGGREGATE_HOURLY_CHECKPOINT_FILE); - sleepInterval = metricsConf.getLong(HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL, - 3600000l); + sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong + (HOST_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l)); checkpointCutOffMultiplier = metricsConf.getInt(HOST_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2); } @@ -181,8 +182,8 @@ public class TimelineMetricAggregatorHourly extends AbstractTimelineAggregator { } @Override - protected Long getSleepInterval() { - return sleepInterval; + protected Long getSleepIntervalMillis() { + return sleepIntervalMillis; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java index 274d0b5..ac9d12e 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricAggregatorMinute.java @@ -32,6 +32,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.Condition; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_METRIC_AGGREGATE_ONLY_SQL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME; @@ -48,7 +49,7 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator { private static final String MINUTE_AGGREGATE_CHECKPOINT_FILE = "timeline-metrics-host-aggregator-checkpoint"; private final String checkpointLocation; - private final Long sleepInterval; + private final Long sleepIntervalMillis; private final Integer checkpointCutOffMultiplier; public TimelineMetricAggregatorMinute(PhoenixHBaseAccessor hBaseAccessor, @@ -61,7 +62,8 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator { checkpointLocation = FilenameUtils.concat(checkpointDir, MINUTE_AGGREGATE_CHECKPOINT_FILE); - sleepInterval = metricsConf.getLong(HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300000l); // 5 mins + sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong + (HOST_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 300l)); // 5 mins checkpointCutOffMultiplier = metricsConf.getInt(HOST_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 3); } @@ -163,8 +165,8 @@ public class TimelineMetricAggregatorMinute extends AbstractTimelineAggregator { } @Override - protected Long getSleepInterval() { - return sleepInterval; + protected Long getSleepIntervalMillis() { + return sleepIntervalMillis; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java index 080703e..c52451e 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregator.java @@ -34,6 +34,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.Condition; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.GET_METRIC_SQL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.PhoenixTransactSQL.METRICS_RECORD_TABLE_NAME; @@ -54,8 +55,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator private static final String CLUSTER_AGGREGATOR_CHECKPOINT_FILE = "timeline-metrics-cluster-aggregator-checkpoint"; private final String checkpointLocation; - private final Long sleepInterval; - public final int timeSliceInterval; + private final Long sleepIntervalMillis; + public final int timeSliceIntervalMillis; private final Integer checkpointCutOffMultiplier; public TimelineMetricClusterAggregator(PhoenixHBaseAccessor hBaseAccessor, @@ -68,8 +69,10 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator checkpointLocation = FilenameUtils.concat(checkpointDir, CLUSTER_AGGREGATOR_CHECKPOINT_FILE); - sleepInterval = metricsConf.getLong(CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 120000l); - timeSliceInterval = metricsConf.getInt(CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 15000); + sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong + (CLUSTER_AGGREGATOR_MINUTE_SLEEP_INTERVAL, 120l)); + timeSliceIntervalMillis = (int)SECONDS.toMillis(metricsConf.getInt + (CLUSTER_AGGREGATOR_TIMESLICE_INTERVAL, 15)); checkpointCutOffMultiplier = metricsConf.getInt(CLUSTER_AGGREGATOR_MINUTE_CHECKPOINT_CUTOFF_MULTIPLIER, 2); } @@ -108,8 +111,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator // Create time slices long sliceStartTime = startTime; while (sliceStartTime < endTime) { - timeSlices.add(new Long[] { sliceStartTime, sliceStartTime + timeSliceInterval}); - sliceStartTime += timeSliceInterval; + timeSlices.add(new Long[] { sliceStartTime, sliceStartTime + timeSliceIntervalMillis}); + sliceStartTime += timeSliceIntervalMillis; } while (rs.next()) { @@ -172,8 +175,8 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator } @Override - protected Long getSleepInterval() { - return sleepInterval; + protected Long getSleepIntervalMillis() { + return sleepIntervalMillis; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java index c380adb..e886b71 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricClusterAggregatorHourly.java @@ -32,6 +32,7 @@ import java.util.Date; import java.util.HashMap; import java.util.Map; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics .timeline.PhoenixHBaseAccessor.getMetricClusterAggregateFromResultSet; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics @@ -60,9 +61,9 @@ public class TimelineMetricClusterAggregatorHourly extends private static final String CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE = "timeline-metrics-cluster-aggregator-hourly-checkpoint"; private final String checkpointLocation; - private final long sleepInterval; + private final long sleepIntervalMillis; private final Integer checkpointCutOffMultiplier; - private long checkpointCutOffInterval; + private long checkpointCutOffIntervalMillis; public TimelineMetricClusterAggregatorHourly( PhoenixHBaseAccessor hBaseAccessor, Configuration metricsConf) { @@ -74,9 +75,9 @@ public class TimelineMetricClusterAggregatorHourly extends checkpointLocation = FilenameUtils.concat(checkpointDir, CLUSTER_AGGREGATOR_HOURLY_CHECKPOINT_FILE); - sleepInterval = metricsConf.getLong - (CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600000l); - checkpointCutOffInterval = 7200000l; + sleepIntervalMillis = SECONDS.toMillis(metricsConf.getLong + (CLUSTER_AGGREGATOR_HOUR_SLEEP_INTERVAL, 3600l)); + checkpointCutOffIntervalMillis = 7200000l; checkpointCutOffMultiplier = metricsConf.getInt (CLUSTER_AGGREGATOR_HOUR_CHECKPOINT_CUTOFF_MULTIPLIER, 2); } @@ -202,8 +203,8 @@ public class TimelineMetricClusterAggregatorHourly extends } @Override - protected Long getSleepInterval() { - return sleepInterval; + protected Long getSleepIntervalMillis() { + return sleepIntervalMillis; } @Override @@ -212,8 +213,8 @@ public class TimelineMetricClusterAggregatorHourly extends } @Override - protected Long getCheckpointCutOffInterval() { - return checkpointCutOffInterval; + protected Long getCheckpointCutOffIntervalMillis() { + return checkpointCutOffIntervalMillis; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/06333585/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java index dd952b4..6b19847 100644 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java +++ b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricConfiguration.java @@ -21,7 +21,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; /** - * Configuration class that reads properties from ams-site.xml + * Configuration class that reads properties from ams-site.xml. All values + * for time or intervals are given in seconds. */ @InterfaceAudience.Public @InterfaceStability.Evolving
