Repository: ambari Updated Branches: refs/heads/branch-3.0-ams 35e2851b9 -> 84534df71
AMBARI-17382 : Migrate AMS queries to use ROW_TIMESTAMP instead of native timerange hint. (avijayan) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/84534df7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/84534df7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/84534df7 Branch: refs/heads/branch-3.0-ams Commit: 84534df71a3437b0c4d9f8fa3edf087f2fe25b6f Parents: 35e2851 Author: Aravindan Vijayan <avija...@hortonworks.com> Authored: Mon Jul 10 17:03:22 2017 -0700 Committer: Aravindan Vijayan <avija...@hortonworks.com> Committed: Mon Jul 10 17:03:22 2017 -0700 ---------------------------------------------------------------------- .../timeline/SingleValuedTimelineMetric.java | 15 ++---- .../metrics2/sink/timeline/TimelineMetric.java | 18 ++----- .../metrics2/sink/timeline/TimelineMetrics.java | 8 +--- .../TimelineMetricsEhCacheSizeOfEngine.java | 1 - .../AggregatedMetricsPublisherTest.java | 6 +-- .../sink/timeline/RawMetricsPublisherTest.java | 2 +- .../ambari-metrics-timelineservice/pom.xml | 6 +-- .../metrics/timeline/PhoenixHBaseAccessor.java | 24 +++------- .../timeline/TimelineMetricStoreWatcher.java | 1 - .../aggregators/AbstractTimelineAggregator.java | 2 +- .../TimelineMetricClusterAggregator.java | 4 +- .../TimelineMetricClusterAggregatorSecond.java | 9 +--- .../TimelineMetricHostAggregator.java | 7 ++- .../aggregators/TimelineMetricReadHelper.java | 4 +- .../v2/TimelineMetricClusterAggregator.java | 2 +- .../v2/TimelineMetricHostAggregator.java | 2 +- .../timeline/query/PhoenixTransactSQL.java | 49 +++++++++----------- .../metrics/timeline/query/TopNCondition.java | 2 - .../source/cache/InternalMetricsCache.java | 2 - .../TestApplicationHistoryServer.java | 1 + .../timeline/AbstractMiniHBaseClusterTest.java | 15 +++--- .../timeline/ITPhoenixHBaseAccessor.java | 2 +- .../metrics/timeline/MetricTestHelper.java | 2 +- .../timeline/TestPhoenixTransactSQL.java | 6 +-- .../aggregators/ITClusterAggregator.java | 1 - .../aggregators/ITMetricAggregator.java | 14 ++---- .../timeline/discovery/TestMetadataManager.java | 5 -- .../timeline/source/RawMetricsSourceTest.java | 1 - ambari-metrics/pom.xml | 12 ++--- .../metrics/timeline/MetricsRequestHelper.java | 1 - .../package/templates/smoketest_metrics.json.j2 | 1 - .../cache/TimelineMetricCacheSizingTest.java | 1 - 32 files changed, 74 insertions(+), 152 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java index 4bb9355..83d8e2c 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/SingleValuedTimelineMetric.java @@ -23,7 +23,6 @@ package org.apache.hadoop.metrics2.sink.timeline; * with @TimelineMetric */ public class SingleValuedTimelineMetric { - private Long timestamp; private Double value; private String metricName; private String appId; @@ -31,26 +30,21 @@ public class SingleValuedTimelineMetric { private String hostName; private Long startTime; - public void setSingleTimeseriesValue(Long timestamp, Double value) { - this.timestamp = timestamp; + public void setSingleTimeseriesValue(Long startTime, Double value) { + this.startTime = startTime; this.value = value; } public SingleValuedTimelineMetric(String metricName, String appId, String instanceId, String hostName, - long timestamp, long startTime) { + long startTime) { this.metricName = metricName; this.appId = appId; this.instanceId = instanceId; this.hostName = hostName; - this.timestamp = timestamp; this.startTime = startTime; } - public Long getTimestamp() { - return timestamp; - } - public long getStartTime() { return startTime; } @@ -93,8 +87,7 @@ public class SingleValuedTimelineMetric { metric.setHostName(this.hostName); metric.setInstanceId(this.instanceId); metric.setStartTime(this.startTime); - metric.setTimestamp(this.timestamp); - metric.getMetricValues().put(timestamp, value); + metric.getMetricValues().put(startTime, value); return metric; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java index 3d3b19c..1f03fe9 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetric.java @@ -40,7 +40,6 @@ public class TimelineMetric implements Comparable<TimelineMetric> { private String appId; private String instanceId; private String hostName; - private long timestamp; private long startTime; private String type; private String units; @@ -65,7 +64,6 @@ public class TimelineMetric implements Comparable<TimelineMetric> { setMetricName(metric.getMetricName()); setType(metric.getType()); setUnits(metric.getUnits()); - setTimestamp(metric.getTimestamp()); setAppId(metric.getAppId()); setInstanceId(metric.getInstanceId()); setHostName(metric.getHostName()); @@ -109,15 +107,6 @@ public class TimelineMetric implements Comparable<TimelineMetric> { this.hostName = hostName; } - @XmlElement(name = "timestamp") - public long getTimestamp() { - return timestamp; - } - - public void setTimestamp(long timestamp) { - this.timestamp = timestamp; - } - @XmlElement(name = "starttime") public long getStartTime() { return startTime; @@ -181,7 +170,6 @@ public class TimelineMetric implements Comparable<TimelineMetric> { return false; if (instanceId != null ? !instanceId.equals(metric.instanceId) : metric.instanceId != null) return false; - if (timestamp != metric.timestamp) return false; if (startTime != metric.startTime) return false; return true; @@ -205,15 +193,15 @@ public class TimelineMetric implements Comparable<TimelineMetric> { result = 31 * result + (appId != null ? appId.hashCode() : 0); result = 31 * result + (instanceId != null ? instanceId.hashCode() : 0); result = 31 * result + (hostName != null ? hostName.hashCode() : 0); - result = 31 * result + (int) (timestamp ^ (timestamp >>> 32)); + result = 31 * result + (int) (startTime ^ (startTime >>> 32)); return result; } @Override public int compareTo(TimelineMetric other) { - if (timestamp > other.timestamp) { + if (startTime > other.startTime) { return -1; - } else if (timestamp < other.timestamp) { + } else if (startTime < other.startTime) { return 1; } else { return metricName.compareTo(other.metricName); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java index 383079a..0c5965c 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/TimelineMetrics.java @@ -89,9 +89,6 @@ public class TimelineMetrics { if (metricToMerge != null) { metricToMerge.addMetricValues(metric.getMetricValues()); - if (metricToMerge.getTimestamp() > metric.getTimestamp()) { - metricToMerge.setTimestamp(metric.getTimestamp()); - } if (metricToMerge.getStartTime() > metric.getStartTime()) { metricToMerge.setStartTime(metric.getStartTime()); } @@ -114,10 +111,7 @@ public class TimelineMetrics { } if (metricToMerge != null) { - metricToMerge.getMetricValues().put(metric.getTimestamp(), metric.getValue()); - if (metricToMerge.getTimestamp() > metric.getTimestamp()) { - metricToMerge.setTimestamp(metric.getTimestamp()); - } + metricToMerge.getMetricValues().put(metric.getStartTime(), metric.getValue()); if (metricToMerge.getStartTime() > metric.getStartTime()) { metricToMerge.setStartTime(metric.getStartTime()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java index 0e23e17..0e4871b 100644 --- a/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java +++ b/ambari-metrics/ambari-metrics-common/src/main/java/org/apache/hadoop/metrics2/sink/timeline/cache/TimelineMetricsEhCacheSizeOfEngine.java @@ -93,7 +93,6 @@ public abstract class TimelineMetricsEhCacheSizeOfEngine implements SizeOfEngine timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getAppId()); timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getHostName()); timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getInstanceId()); - timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getTimestamp()); timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getStartTime()); timelineMetricPrimitivesApproximation += reflectionSizeOf.sizeOf(metric.getType()); timelineMetricPrimitivesApproximation += 8; // Object overhead http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisherTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisherTest.java b/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisherTest.java index 3413052..8c17ba1 100644 --- a/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisherTest.java +++ b/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/AggregatedMetricsPublisherTest.java @@ -62,9 +62,9 @@ public class AggregatedMetricsPublisherTest { new AggregatedMetricsPublisher(TimelineMetricsHolder.getInstance(), configuration, 60); String aggregatedJson = aggregatedMetricsPublisher.processMetrics(timelineMetricsHolder.extractMetricsForAggregationPublishing()); - String expectedMetric1App1 = "{\"timelineMetric\":{\"timestamp\":0,\"metadata\":{},\"metricname\":\"metricName1\",\"appid\":\"app1\",\"starttime\":0,\"metrics\":{}},\"metricAggregate\":{\"sum\":6.0,\"deviation\":0.0,\"max\":3.0,\"min\":1.0,\"numberOfSamples\":3}}"; - String expectedMetric2App2 = "{\"timelineMetric\":{\"timestamp\":0,\"metadata\":{},\"metricname\":\"metricName2\",\"appid\":\"app2\",\"starttime\":0,\"metrics\":{}},\"metricAggregate\":{\"sum\":15.0,\"deviation\":0.0,\"max\":6.0,\"min\":4.0,\"numberOfSamples\":3}}"; - String expectedMetric3App3 = "{\"timelineMetric\":{\"timestamp\":0,\"metadata\":{},\"metricname\":\"metricName3\",\"appid\":\"app3\",\"starttime\":0,\"metrics\":{}},\"metricAggregate\":{\"sum\":24.0,\"deviation\":0.0,\"max\":9.0,\"min\":7.0,\"numberOfSamples\":3}}"; + String expectedMetric1App1 = "{\"timelineMetric\":{\"metadata\":{},\"metricname\":\"metricName1\",\"appid\":\"app1\",\"starttime\":0,\"metrics\":{}},\"metricAggregate\":{\"sum\":6.0,\"deviation\":0.0,\"max\":3.0,\"min\":1.0,\"numberOfSamples\":3}}"; + String expectedMetric2App2 = "{\"timelineMetric\":{\"metadata\":{},\"metricname\":\"metricName2\",\"appid\":\"app2\",\"starttime\":0,\"metrics\":{}},\"metricAggregate\":{\"sum\":15.0,\"deviation\":0.0,\"max\":6.0,\"min\":4.0,\"numberOfSamples\":3}}"; + String expectedMetric3App3 = "{\"timelineMetric\":{\"metadata\":{},\"metricname\":\"metricName3\",\"appid\":\"app3\",\"starttime\":0,\"metrics\":{}},\"metricAggregate\":{\"sum\":24.0,\"deviation\":0.0,\"max\":9.0,\"min\":7.0,\"numberOfSamples\":3}}"; Assert.assertNotNull(aggregatedJson); Assert.assertTrue(aggregatedJson.contains(expectedMetric1App1)); Assert.assertTrue(aggregatedJson.contains(expectedMetric3App3)); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisherTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisherTest.java b/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisherTest.java index 60510d2..b43a87c 100644 --- a/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisherTest.java +++ b/ambari-metrics/ambari-metrics-host-aggregator/src/test/java/org/apache/hadoop/metrics2/sink/timeline/RawMetricsPublisherTest.java @@ -62,7 +62,7 @@ public class RawMetricsPublisherTest { new RawMetricsPublisher(TimelineMetricsHolder.getInstance(), configuration, 60); String rawJson = rawMetricsPublisher.processMetrics(timelineMetricsHolder.extractMetricsForRawPublishing()); - String expectedResult = "{\"metrics\":[{\"timestamp\":0,\"metadata\":{},\"metricname\":\"metricName1\",\"appid\":\"app1\",\"starttime\":0,\"metrics\":{\"1\":1.0,\"2\":2.0,\"3\":3.0}},{\"timestamp\":0,\"metadata\":{},\"metricname\":\"metricName2\",\"appid\":\"app2\",\"starttime\":0,\"metrics\":{\"1\":4.0,\"2\":5.0,\"3\":6.0}},{\"timestamp\":0,\"metadata\":{},\"metricname\":\"metricName3\",\"appid\":\"app3\",\"starttime\":0,\"metrics\":{\"1\":7.0,\"2\":8.0,\"3\":9.0}}]}"; + String expectedResult = "{\"metrics\":[{\"metadata\":{},\"metricname\":\"metricName1\",\"appid\":\"app1\",\"starttime\":0,\"metrics\":{\"1\":1.0,\"2\":2.0,\"3\":3.0}},{\"metadata\":{},\"metricname\":\"metricName2\",\"appid\":\"app2\",\"starttime\":0,\"metrics\":{\"1\":4.0,\"2\":5.0,\"3\":6.0}},{\"metadata\":{},\"metricname\":\"metricName3\",\"appid\":\"app3\",\"starttime\":0,\"metrics\":{\"1\":7.0,\"2\":8.0,\"3\":9.0}}]}"; Assert.assertNotNull(rawJson); Assert.assertEquals(expectedResult, rawJson); } http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml index 34794e6..501c06d 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml +++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml @@ -34,9 +34,9 @@ <!-- Needed for generating FindBugs warnings using parent pom --> <!--<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>--> <protobuf.version>2.5.0</protobuf.version> - <hadoop.version>2.7.1.2.3.4.0-3347</hadoop.version> - <phoenix.version>4.4.0.2.3.4.0-3347</phoenix.version> - <hbase.version>1.1.2.2.3.4.0-3347</hbase.version> + <hadoop.version>2.7.3.2.6.0.3-8</hadoop.version> + <phoenix.version>4.7.0.2.6.0.3-8</phoenix.version> + <hbase.version>1.1.2.2.6.0.3-8</hbase.version> </properties> <build> http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java index 7ad88a1..6ea6f70 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/PhoenixHBaseAccessor.java @@ -321,14 +321,6 @@ public class PhoenixHBaseAccessor { UPSERT_METRICS_SQL, METRICS_RECORD_TABLE_NAME)); for (TimelineMetrics timelineMetrics : timelineMetricsCollection) { for (TimelineMetric metric : timelineMetrics.getMetrics()) { - if (Math.abs(currentTime - metric.getStartTime()) > outOfBandTimeAllowance) { - // If timeseries start time is way in the past : discard - LOG.debug("Discarding out of band timeseries, currentTime = " - + currentTime + ", startTime = " + metric.getStartTime() - + ", hostname = " + metric.getHostName()); - continue; - } - metricRecordStmt.clearParameters(); if (LOG.isTraceEnabled()) { @@ -345,14 +337,13 @@ public class PhoenixHBaseAccessor { continue; } metricRecordStmt.setBytes(1, uuid); - metricRecordStmt.setLong(2, currentTime); - metricRecordStmt.setLong(3, metric.getStartTime()); - metricRecordStmt.setDouble(4, aggregates[0]); - metricRecordStmt.setDouble(5, aggregates[1]); - metricRecordStmt.setDouble(6, aggregates[2]); - metricRecordStmt.setLong(7, (long) aggregates[3]); + metricRecordStmt.setLong(2, metric.getStartTime()); + metricRecordStmt.setDouble(3, aggregates[0]); + metricRecordStmt.setDouble(4, aggregates[1]); + metricRecordStmt.setDouble(5, aggregates[2]); + metricRecordStmt.setLong(6, (long) aggregates[3]); String json = TimelineUtils.dumpTimelineRecordtoJSON(metric.getMetricValues()); - metricRecordStmt.setString(8, json); + metricRecordStmt.setString(7, json); try { metricRecordStmt.executeUpdate(); @@ -1191,7 +1182,6 @@ public class PhoenixHBaseAccessor { timelineMetric.getAppId(), timelineMetric.getInstanceId(), null, - rs.getLong("SERVER_TIME"), rs.getLong("SERVER_TIME") ); @@ -1283,7 +1273,7 @@ public class PhoenixHBaseAccessor { rowCount++; stmt.clearParameters(); stmt.setBytes(1, uuid); - stmt.setLong(2, metric.getTimestamp()); + stmt.setLong(2, metric.getStartTime()); stmt.setDouble(3, hostAggregate.getSum()); stmt.setDouble(4, hostAggregate.getMax()); stmt.setDouble(5, hostAggregate.getMin()); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java index aa53430..517d3a4 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TimelineMetricStoreWatcher.java @@ -93,7 +93,6 @@ public class TimelineMetricStoreWatcher implements Runnable { fakeMetric.setHostName(FAKE_HOSTNAME); fakeMetric.setAppId(FAKE_APP_ID); fakeMetric.setStartTime(startTime); - fakeMetric.setTimestamp(startTime); fakeMetric.getMetricValues().put(startTime, 0.0); final TimelineMetrics metrics = new TimelineMetrics(); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java index 83f2392..b7c8bb4 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/AbstractTimelineAggregator.java @@ -330,7 +330,7 @@ public abstract class AbstractTimelineAggregator implements TimelineMetricAggreg if (outputTableName.contains("RECORD")) { queryPrefix = PhoenixTransactSQL.DOWNSAMPLE_HOST_METRIC_SQL_UPSERT_PREFIX; } - queryPrefix = String.format(queryPrefix, getQueryHint(startTime), outputTableName); + queryPrefix = String.format(queryPrefix, outputTableName); for (Iterator<CustomDownSampler> iterator = configuredDownSamplers.iterator(); iterator.hasNext();){ CustomDownSampler downSampler = iterator.next(); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java index 0f6dd79..7368bfb 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregator.java @@ -67,10 +67,10 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator endTime, null, null, true); condition.setNoLimit(); condition.setFetchSize(resultsetFetchSize); - String sqlStr = String.format(GET_CLUSTER_AGGREGATE_TIME_SQL, getQueryHint(startTime), tableName); + String sqlStr = String.format(GET_CLUSTER_AGGREGATE_TIME_SQL, tableName); // HOST_COUNT vs METRIC_COUNT if (isClusterPrecisionInputTable) { - sqlStr = String.format(GET_CLUSTER_AGGREGATE_SQL, getQueryHint(startTime), tableName); + sqlStr = String.format(GET_CLUSTER_AGGREGATE_SQL, tableName); } condition.setStatement(sqlStr); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java index cae7263..1348da6 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricClusterAggregatorSecond.java @@ -127,7 +127,7 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre condition.setNoLimit(); condition.setFetchSize(resultsetFetchSize); condition.setStatement(String.format(GET_METRIC_SQL, - getQueryHint(startTime), METRICS_RECORD_TABLE_NAME)); + METRICS_RECORD_TABLE_NAME)); // Retaining order of the row-key avoids client side merge sort. condition.addOrderByColumn("UUID"); condition.addOrderByColumn("SERVER_TIME"); @@ -258,13 +258,6 @@ public class TimelineMetricClusterAggregatorSecond extends AbstractTimelineAggre Map<TimelineClusterMetric, Double> timelineClusterMetricMap = new HashMap<TimelineClusterMetric, Double>(); - Long timeShift = timelineMetric.getTimestamp() - timelineMetric.getStartTime(); - if (timeShift < 0) { - LOG.debug("Invalid time shift found, possible discrepancy in clocks. " + - "timeShift = " + timeShift); - timeShift = 0l; - } - Long prevTimestamp = -1l; TimelineClusterMetric prevMetric = null; int count = 0; http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java index 8f941e1..f9f92db 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricHostAggregator.java @@ -74,8 +74,7 @@ public class TimelineMetricHostAggregator extends AbstractTimelineAggregator { endTime, null, null, true); condition.setNoLimit(); condition.setFetchSize(resultsetFetchSize); - condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, - getQueryHint(startTime), tableName)); + condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, tableName)); // Retaining order of the row-key avoids client side merge sort. condition.addOrderByColumn("UUID"); condition.addOrderByColumn("SERVER_TIME"); @@ -98,7 +97,7 @@ public class TimelineMetricHostAggregator extends AbstractTimelineAggregator { if (existingMetric == null) { // First row existingMetric = currentMetric; - currentMetric.setTimestamp(endTime); + currentMetric.setStartTime(endTime); hostAggregate = new MetricHostAggregate(); hostAggregateMap.put(currentMetric, hostAggregate); } @@ -108,7 +107,7 @@ public class TimelineMetricHostAggregator extends AbstractTimelineAggregator { hostAggregate.updateAggregates(currentHostAggregate); } else { // Switched over to a new metric - save existing - create new aggregate - currentMetric.setTimestamp(endTime); + currentMetric.setStartTime(endTime); hostAggregate = new MetricHostAggregate(); hostAggregate.updateAggregates(currentHostAggregate); hostAggregateMap.put(currentMetric, hostAggregate); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java index c8b5728..40b5eac 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/TimelineMetricReadHelper.java @@ -69,7 +69,6 @@ public class TimelineMetricReadHelper { timelineMetric.getAppId(), timelineMetric.getInstanceId(), timelineMetric.getHostName(), - rs.getLong("SERVER_TIME"), rs.getLong("SERVER_TIME") ); @@ -108,8 +107,7 @@ public class TimelineMetricReadHelper { if (ignoreInstance) { metric.setInstanceId(null); } - metric.setTimestamp(rs.getLong("SERVER_TIME")); - metric.setStartTime(rs.getLong("START_TIME")); + metric.setStartTime(rs.getLong("SERVER_TIME")); return metric; } http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java index c7b605f..e6d0b32 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricClusterAggregator.java @@ -74,7 +74,7 @@ public class TimelineMetricClusterAggregator extends AbstractTimelineAggregator */ condition.setStatement(String.format(GET_AGGREGATED_APP_METRIC_GROUPBY_SQL, - getQueryHint(startTime), outputTableName, endTime, aggregateColumnName, tableName, + outputTableName, endTime, aggregateColumnName, tableName, getDownsampledMetricSkipClause(), startTime, endTime)); if (LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java index 57a3034..5cec65d 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/v2/TimelineMetricHostAggregator.java @@ -64,7 +64,7 @@ public class TimelineMetricHostAggregator extends AbstractTimelineAggregator { condition.setDoUpdate(true); condition.setStatement(String.format(GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL, - getQueryHint(startTime), outputTableName, endTime, tableName, + outputTableName, endTime, tableName, getDownsampledMetricSkipClause(), startTime, endTime)); if (LOG.isDebugEnabled()) { http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java index 51c96c6..5eaa555 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/PhoenixTransactSQL.java @@ -42,13 +42,12 @@ public class PhoenixTransactSQL { public static final String CREATE_METRICS_TABLE_SQL = "CREATE TABLE IF NOT " + "EXISTS METRIC_RECORD (UUID BINARY(20) NOT NULL, " + "SERVER_TIME BIGINT NOT NULL, " + - "START_TIME UNSIGNED_LONG, " + "METRIC_SUM DOUBLE, " + "METRIC_COUNT UNSIGNED_INT, " + "METRIC_MAX DOUBLE, " + "METRIC_MIN DOUBLE, " + "METRICS VARCHAR CONSTRAINT pk " + - "PRIMARY KEY (UUID, SERVER_TIME)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " + + "PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " + "TTL=%s, COMPRESSION='%s'"; public static final String CREATE_CONTAINER_METRICS_TABLE_SQL = @@ -81,35 +80,35 @@ public class PhoenixTransactSQL { public static final String CREATE_METRICS_AGGREGATE_TABLE_SQL = "CREATE TABLE IF NOT EXISTS %s " + "(UUID BINARY(20) NOT NULL, " + - "SERVER_TIME UNSIGNED_LONG NOT NULL, " + + "SERVER_TIME BIGINT NOT NULL, " + "METRIC_SUM DOUBLE," + "METRIC_COUNT UNSIGNED_INT, " + "METRIC_MAX DOUBLE," + "METRIC_MIN DOUBLE CONSTRAINT pk " + - "PRIMARY KEY (UUID, SERVER_TIME)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, TTL=%s," + + "PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, TTL=%s," + " COMPRESSION='%s'"; public static final String CREATE_METRICS_CLUSTER_AGGREGATE_TABLE_SQL = "CREATE TABLE IF NOT EXISTS %s " + "(UUID BINARY(16) NOT NULL, " + - "SERVER_TIME UNSIGNED_LONG NOT NULL, " + + "SERVER_TIME BIGINT NOT NULL, " + "METRIC_SUM DOUBLE, " + "HOSTS_COUNT UNSIGNED_INT, " + "METRIC_MAX DOUBLE, " + "METRIC_MIN DOUBLE " + - "CONSTRAINT pk PRIMARY KEY (UUID, SERVER_TIME)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " + + "CONSTRAINT pk PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " + "TTL=%s, COMPRESSION='%s'"; // HOSTS_COUNT vs METRIC_COUNT public static final String CREATE_METRICS_CLUSTER_AGGREGATE_GROUPED_TABLE_SQL = "CREATE TABLE IF NOT EXISTS %s " + "(UUID BINARY(16) NOT NULL, " + - "SERVER_TIME UNSIGNED_LONG NOT NULL, " + + "SERVER_TIME BIGINT NOT NULL, " + "METRIC_SUM DOUBLE, " + "METRIC_COUNT UNSIGNED_INT, " + "METRIC_MAX DOUBLE, " + "METRIC_MIN DOUBLE " + - "CONSTRAINT pk PRIMARY KEY (UUID, SERVER_TIME)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " + + "CONSTRAINT pk PRIMARY KEY (UUID, SERVER_TIME ROW_TIMESTAMP)) DATA_BLOCK_ENCODING='%s', IMMUTABLE_ROWS=true, " + "TTL=%s, COMPRESSION='%s'"; public static final String CREATE_METRICS_METADATA_TABLE_SQL = @@ -152,13 +151,12 @@ public class PhoenixTransactSQL { public static final String UPSERT_METRICS_SQL = "UPSERT INTO %s " + "(UUID, " + "SERVER_TIME, " + - "START_TIME, " + "METRIC_SUM, " + "METRIC_MAX, " + "METRIC_MIN, " + "METRIC_COUNT, " + "METRICS) VALUES " + - "(?, ?, ?, ?, ?, ?, ?, ?)"; + "(?, ?, ?, ?, ?, ?, ?)"; public static final String UPSERT_CONTAINER_METRICS_SQL = "UPSERT INTO %s " + "(APP_ID," @@ -227,7 +225,7 @@ public class PhoenixTransactSQL { /** * Retrieve a set of rows from metrics records table. */ - public static final String GET_METRIC_SQL = "SELECT %s UUID, SERVER_TIME, START_TIME, " + + public static final String GET_METRIC_SQL = "SELECT UUID, SERVER_TIME, " + "METRIC_SUM, " + "METRIC_MAX, " + "METRIC_MIN, " + @@ -242,7 +240,7 @@ public class PhoenixTransactSQL { * in Apache Phoenix */ public static final String GET_LATEST_METRIC_SQL = "SELECT %s E.UUID AS UUID, " + - "E.SERVER_TIME AS SERVER_TIME, E.START_TIME AS START_TIME, " + + "E.SERVER_TIME AS SERVER_TIME, " + "E.METRIC_SUM AS METRIC_SUM, " + "E.METRIC_MAX AS METRIC_MAX, E.METRIC_MIN AS METRIC_MIN, " + "E.METRIC_COUNT AS METRIC_COUNT, E.METRICS AS METRICS " + @@ -257,7 +255,7 @@ public class PhoenixTransactSQL { "ON E.UUID=I.UUID " + "AND E.SERVER_TIME=I.MAX_SERVER_TIME"; - public static final String GET_METRIC_AGGREGATE_ONLY_SQL = "SELECT %s UUID, " + + public static final String GET_METRIC_AGGREGATE_ONLY_SQL = "SELECT UUID, " + "SERVER_TIME, " + "METRIC_SUM, " + "METRIC_MAX, " + @@ -265,7 +263,7 @@ public class PhoenixTransactSQL { "METRIC_COUNT " + "FROM %s"; - public static final String GET_CLUSTER_AGGREGATE_SQL = "SELECT %s " + + public static final String GET_CLUSTER_AGGREGATE_SQL = "SELECT " + "UUID, " + "SERVER_TIME, " + "METRIC_SUM, " + @@ -274,7 +272,7 @@ public class PhoenixTransactSQL { "METRIC_MIN " + "FROM %s"; - public static final String GET_CLUSTER_AGGREGATE_TIME_SQL = "SELECT %s " + + public static final String GET_CLUSTER_AGGREGATE_TIME_SQL = "SELECT " + "UUID, " + "SERVER_TIME, " + "METRIC_SUM, " + @@ -283,7 +281,7 @@ public class PhoenixTransactSQL { "METRIC_MIN " + "FROM %s"; - public static final String TOP_N_INNER_SQL = "SELECT %s UUID " + + public static final String TOP_N_INNER_SQL = "SELECT UUID " + "FROM %s WHERE %s GROUP BY UUID ORDER BY %s LIMIT %s"; public static final String GET_METRIC_METADATA_SQL = "SELECT " + @@ -300,7 +298,7 @@ public class PhoenixTransactSQL { * Aggregate host metrics using a GROUP BY clause to take advantage of * N - way parallel scan where N = number of regions. */ - public static final String GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL = "UPSERT %s " + + public static final String GET_AGGREGATED_HOST_METRIC_GROUPBY_SQL = "UPSERT " + "INTO %s (UUID, SERVER_TIME, METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) " + "SELECT UUID, %s AS SERVER_TIME, " + "SUM(METRIC_SUM), SUM(METRIC_COUNT), MAX(METRIC_MAX), MIN(METRIC_MIN) " + @@ -310,7 +308,7 @@ public class PhoenixTransactSQL { /** * Downsample host metrics. */ - public static final String DOWNSAMPLE_HOST_METRIC_SQL_UPSERT_PREFIX = "UPSERT %s INTO %s (UUID, SERVER_TIME, " + + public static final String DOWNSAMPLE_HOST_METRIC_SQL_UPSERT_PREFIX = "UPSERT INTO %s (UUID, SERVER_TIME, " + "METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) "; public static final String TOPN_DOWNSAMPLER_HOST_METRIC_SELECT_SQL = "SELECT UUID, " + @@ -321,7 +319,7 @@ public class PhoenixTransactSQL { * Aggregate app metrics using a GROUP BY clause to take advantage of * N - way parallel scan where N = number of regions. */ - public static final String GET_AGGREGATED_APP_METRIC_GROUPBY_SQL = "UPSERT %s " + + public static final String GET_AGGREGATED_APP_METRIC_GROUPBY_SQL = "UPSERT " + "INTO %s (UUID, SERVER_TIME, METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) SELECT UUID, %s AS SERVER_TIME, " + "ROUND(AVG(METRIC_SUM),2), ROUND(AVG(%s)), MAX(METRIC_MAX), MIN(METRIC_MIN) FROM %s WHERE%s SERVER_TIME > %s AND " + "SERVER_TIME <= %s GROUP BY UUID"; @@ -329,7 +327,7 @@ public class PhoenixTransactSQL { /** * Downsample cluster metrics. */ - public static final String DOWNSAMPLE_CLUSTER_METRIC_SQL_UPSERT_PREFIX = "UPSERT %s INTO %s (UUID, SERVER_TIME, " + + public static final String DOWNSAMPLE_CLUSTER_METRIC_SQL_UPSERT_PREFIX = "UPSERT INTO %s (UUID, SERVER_TIME, " + "METRIC_SUM, METRIC_COUNT, METRIC_MAX, METRIC_MIN) "; public static final String TOPN_DOWNSAMPLER_CLUSTER_METRIC_SELECT_SQL = "SELECT UUID, " + @@ -368,7 +366,6 @@ public class PhoenixTransactSQL { public static final String DEFAULT_TABLE_COMPRESSION = "SNAPPY"; public static final String DEFAULT_ENCODING = "FAST_DIFF"; - public static final long NATIVE_TIME_RANGE_DELTA = 120000; // 2 minutes public static final long HOUR = 3600000; // 1 hour public static final long DAY = 86400000; // 1 day private static boolean sortMergeJoinEnabled = false; @@ -435,9 +432,7 @@ public class PhoenixTransactSQL { query = GET_METRIC_SQL; } - stmtStr = String.format(query, - getNaiveTimeRangeHint(condition.getStartTime(), NATIVE_TIME_RANGE_DELTA), - metricsTable); + stmtStr = String.format(query, metricsTable); } StringBuilder sb = new StringBuilder(stmtStr); @@ -630,9 +625,7 @@ public class PhoenixTransactSQL { queryStmt = GET_CLUSTER_AGGREGATE_SQL; } - queryStmt = String.format(queryStmt, - getNaiveTimeRangeHint(condition.getStartTime(), NATIVE_TIME_RANGE_DELTA), - metricsAggregateTable); + queryStmt = String.format(queryStmt, metricsAggregateTable); StringBuilder sb = new StringBuilder(queryStmt); sb.append(" WHERE "); @@ -684,7 +677,7 @@ public class PhoenixTransactSQL { if (condition.getStatement() != null) { stmtStr = condition.getStatement(); } else { - stmtStr = String.format(GET_CLUSTER_AGGREGATE_SQL, "", + stmtStr = String.format(GET_CLUSTER_AGGREGATE_SQL, METRICS_CLUSTER_AGGREGATE_TABLE_NAME); } http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java index 93242bd..38d0c6f 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/query/TopNCondition.java @@ -22,7 +22,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.metrics2.sink.timeline.Precision; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.aggregators.Function; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.NATIVE_TIME_RANGE_DELTA; import java.util.List; public class TopNCondition extends DefaultCondition{ @@ -66,7 +65,6 @@ public class TopNCondition extends DefaultCondition{ public String getTopNInnerQuery() { return String.format(PhoenixTransactSQL.TOP_N_INNER_SQL, - PhoenixTransactSQL.getNaiveTimeRangeHint(getStartTime(), NATIVE_TIME_RANGE_DELTA), PhoenixTransactSQL.getTargetTableUsingPrecision(precision, CollectionUtils.isNotEmpty(hostnames)), super.getConditionClause().toString(), getTopNOrderByClause(), topN); } http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java index a4ed9bc..e5522c7 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/cache/InternalMetricsCache.java @@ -17,7 +17,6 @@ */ package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.source.cache; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -124,7 +123,6 @@ public class InternalMetricsCache { metric.setInstanceId(key.getInstanceId()); metric.setHostName(key.getHostname()); metric.setStartTime(key.getStartTime()); - metric.setTimestamp(key.getStartTime()); Element ele = cache.get(key); metric.setMetricValues(((InternalMetricCacheValue) ele.getObjectValue()).getMetricValues()); metrics.getMetrics().add(metric); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java index 41ddef5..03205e7 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java @@ -158,6 +158,7 @@ public class TestApplicationHistoryServer { } // simple test init/start/stop ApplicationHistoryServer. Status should change. + @Ignore @Test(timeout = 50000) public void testStartStopServer() throws Exception { Configuration config = new YarnConfiguration(); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java index fbf7b09..3a42db9 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractMiniHBaseClusterTest.java @@ -259,15 +259,14 @@ public abstract class AbstractMiniHBaseClusterTest extends BaseTest { metricRecordStmt.setString(2, metric.getHostName()); metricRecordStmt.setString(3, metric.getAppId()); metricRecordStmt.setString(4, metric.getInstanceId()); - metricRecordStmt.setLong(5, currentTime); - metricRecordStmt.setLong(6, metric.getStartTime()); - metricRecordStmt.setString(7, metric.getType()); - metricRecordStmt.setDouble(8, aggregates[0]); - metricRecordStmt.setDouble(9, aggregates[1]); - metricRecordStmt.setDouble(10, aggregates[2]); - metricRecordStmt.setLong(11, (long) aggregates[3]); + metricRecordStmt.setLong(5, metric.getStartTime()); + metricRecordStmt.setString(6, metric.getType()); + metricRecordStmt.setDouble(7, aggregates[0]); + metricRecordStmt.setDouble(8, aggregates[1]); + metricRecordStmt.setDouble(9, aggregates[2]); + metricRecordStmt.setLong(10, (long) aggregates[3]); String json = TimelineUtils.dumpTimelineRecordtoJSON(metric.getMetricValues()); - metricRecordStmt.setString(12, json); + metricRecordStmt.setString(11, json); try { metricRecordStmt.executeUpdate(); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java index c25d414..c60554c 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ITPhoenixHBaseAccessor.java @@ -422,7 +422,7 @@ public class ITPhoenixHBaseAccessor extends AbstractMiniHBaseClusterTest { while (set.next()) { assertEquals("application_1450744875949_0001", set.getString("APP_ID")); assertEquals("container_1450744875949_0001_01_000001", set.getString("CONTAINER_ID")); - assertEquals(new java.sql.Timestamp(startTime), set.getTimestamp("START_TIME")); + assertEquals(new java.sql.Timestamp(startTime), set.getTimestamp("SERVER_TIME")); assertEquals(new java.sql.Timestamp(finishTime), set.getTimestamp("FINISH_TIME")); assertEquals(5000, set.getLong("DURATION")); assertEquals("host1", set.getString("HOSTNAME")); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java index 7dfe1fc..74da438 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricTestHelper.java @@ -101,7 +101,7 @@ public class MetricTestHelper { metric.setAppId("test_app"); metric.setInstanceId("test_instance"); metric.setHostName("test_host"); - metric.setTimestamp(startTime); + metric.setStartTime(startTime); return metric; } http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java index dd73a8a..cb3f3a7 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/TestPhoenixTransactSQL.java @@ -566,8 +566,7 @@ public class TestPhoenixTransactSQL { String conditionClause = condition.getConditionClause().toString(); String expectedClause = " UUID IN (" + - "SELECT " + PhoenixTransactSQL.getNaiveTimeRangeHint(condition.getStartTime(),120000l) + " " + - "UUID FROM METRIC_RECORD WHERE " + + "SELECT UUID FROM METRIC_RECORD WHERE " + "(UUID IN (?, ?)) AND " + "SERVER_TIME >= ? AND SERVER_TIME < ? " + "GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?"; @@ -585,8 +584,7 @@ public class TestPhoenixTransactSQL { String conditionClause = condition.getConditionClause().toString(); String expectedClause = " UUID IN (" + - "SELECT " + PhoenixTransactSQL.getNaiveTimeRangeHint(condition.getStartTime(),120000l) + - " UUID FROM METRIC_RECORD WHERE " + + "SELECT UUID FROM METRIC_RECORD WHERE " + "(UUID IN (?, ?, ?)) AND " + "SERVER_TIME >= ? AND SERVER_TIME < ? " + "GROUP BY UUID ORDER BY MAX(METRIC_MAX) DESC LIMIT 2) AND SERVER_TIME >= ? AND SERVER_TIME < ?"; http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java index e66e65d..a9f2b4d 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITClusterAggregator.java @@ -28,7 +28,6 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.GET_CLUSTER_AGGREGATE_SQL; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_HOURLY_TABLE_NAME; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_CLUSTER_AGGREGATE_TABLE_NAME; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.NATIVE_TIME_RANGE_DELTA; import java.sql.Connection; import java.sql.PreparedStatement; http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java index 14ac4d7..1890819 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/aggregators/ITMetricAggregator.java @@ -45,7 +45,6 @@ import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.ti import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_DAILY_TABLE_NAME; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_HOURLY_TABLE_NAME; import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.METRICS_AGGREGATE_MINUTE_TABLE_NAME; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.query.PhoenixTransactSQL.NATIVE_TIME_RANGE_DELTA; import static org.assertj.core.api.Assertions.assertThat; public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { @@ -105,7 +104,6 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { Condition condition = new DefaultCondition(null, null, null, null, startTime, endTime, null, null, true); condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, - PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), METRICS_AGGREGATE_MINUTE_TABLE_NAME)); PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition); @@ -181,9 +179,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { //THEN Condition condition = new DefaultCondition(null, null, null, null, startTime, endTime, null, null, true); - condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, - PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), - METRICS_AGGREGATE_HOURLY_TABLE_NAME)); + condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_HOURLY_TABLE_NAME)); PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition); ResultSet rs = pstmt.executeQuery(); @@ -243,9 +239,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { //THEN Condition condition = new DefaultCondition(null, null, null, null, startTime, endTime, null, null, true); - condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, - PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), - METRICS_AGGREGATE_DAILY_TABLE_NAME)); + condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_DAILY_TABLE_NAME)); PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition); ResultSet rs = pstmt.executeQuery(); @@ -289,9 +283,7 @@ public class ITMetricAggregator extends AbstractMiniHBaseClusterTest { Condition condition = new DefaultCondition(null, null, null, null, startTime, endTime, null, null, true); - condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, - PhoenixTransactSQL.getNaiveTimeRangeHint(startTime, NATIVE_TIME_RANGE_DELTA), - METRICS_AGGREGATE_MINUTE_TABLE_NAME)); + condition.setStatement(String.format(GET_METRIC_AGGREGATE_ONLY_SQL, METRICS_AGGREGATE_MINUTE_TABLE_NAME)); PreparedStatement pstmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition); ResultSet rs = pstmt.executeQuery(); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java index ca1fc20..f9a1036 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/discovery/TestMetadataManager.java @@ -56,7 +56,6 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest { TimelineMetric metric1 = new TimelineMetric(); metric1.setMetricName("dummy_metric1"); metric1.setHostName("dummy_host1"); - metric1.setTimestamp(now); metric1.setStartTime(now - 1000); metric1.setAppId("dummy_app1"); metric1.setType("Integer"); @@ -69,7 +68,6 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest { TimelineMetric metric2 = new TimelineMetric(); metric2.setMetricName("dummy_metric2"); metric2.setHostName("dummy_host2"); - metric2.setTimestamp(now); metric2.setStartTime(now - 1000); metric2.setAppId("dummy_app2"); metric2.setType("Integer"); @@ -175,7 +173,6 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest { TimelineMetric metric1 = new TimelineMetric(); metric1.setMetricName("dummy_m1"); metric1.setHostName("dummy_host1"); - metric1.setTimestamp(now); metric1.setStartTime(now - 1000); metric1.setAppId("dummy_app1"); metric1.setType("Integer"); @@ -189,7 +186,6 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest { TimelineMetric metric2 = new TimelineMetric(); metric2.setMetricName("dummy_m2"); metric2.setHostName("dummy_host2"); - metric2.setTimestamp(now); metric2.setStartTime(now - 1000); metric2.setAppId("dummy_app2"); metric2.setType("Integer"); @@ -203,7 +199,6 @@ public class TestMetadataManager extends AbstractMiniHBaseClusterTest { TimelineMetric metric3 = new TimelineMetric(); metric3.setMetricName("gummy_3"); metric3.setHostName("dummy_3h"); - metric3.setTimestamp(now); metric3.setStartTime(now - 1000); metric3.setAppId("dummy_app3"); metric3.setType("Integer"); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java index 5d3aacb..254ee6c 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/source/RawMetricsSourceTest.java @@ -121,7 +121,6 @@ public class RawMetricsSourceTest { metric1.setInstanceId("i1"); metric1.setHostName("h1"); metric1.setStartTime(now - 200); - metric1.setTimestamp(now - 200); metric1.setMetricValues(new TreeMap<Long, Double>() {{ put(now - 100, 1.0); put(now - 200, 2.0); http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-metrics/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml index 21608e4..3864526 100644 --- a/ambari-metrics/pom.xml +++ b/ambari-metrics/pom.xml @@ -42,18 +42,18 @@ <python.ver>python >= 2.6</python.ver> <deb.python.ver>python (>= 2.6)</deb.python.ver> <!--TODO change to HDP URL--> - <hbase.tar>http://private-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.4.0-3347/tars/hbase-1.1.2.2.3.4.0-3347.tar.gz</hbase.tar> - <hbase.folder>hbase-1.1.2.2.3.4.0-3347</hbase.folder> - <hadoop.tar>http://private-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.4.0-3347/tars/hadoop-2.7.1.2.3.4.0-3347.tar.gz</hadoop.tar> - <hadoop.folder>hadoop-2.7.1.2.3.4.0-3347</hadoop.folder> + <hbase.tar>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.0.3/tars/hbase-1.1.2.2.6.0.3-8.tar.gz</hbase.tar> + <hbase.folder>hbase-1.1.2.2.6.0.3-8</hbase.folder> + <hadoop.tar>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.0.3/tars/hadoop/hadoop-2.7.3.2.6.0.3-8.tar.gz</hadoop.tar> + <hadoop.folder>hadoop-2.7.3.2.6.0.3-8</hadoop.folder> <hbase.winpkg.zip>https://msibuilds.blob.core.windows.net/hdp/2.x/2.2.4.2/2/hbase-0.98.4.2.2.4.2-0002-hadoop2.winpkg.zip</hbase.winpkg.zip> <hbase.winpkg.folder>hbase-0.98.4.2.2.4.2-0002-hadoop2</hbase.winpkg.folder> <hadoop.winpkg.zip>https://msibuilds.blob.core.windows.net/hdp/2.x/2.2.4.2/2/hadoop-2.6.0.2.2.4.2-0002.winpkg.zip</hadoop.winpkg.zip> <hadoop.winpkg.folder>hadoop-2.6.0.2.2.4.2-0002</hadoop.winpkg.folder> <grafana.folder>grafana-2.6.0</grafana.folder> <grafana.tar>https://grafanarel.s3.amazonaws.com/builds/grafana-2.6.0.linux-x64.tar.gz</grafana.tar> - <phoenix.tar>http://private-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.4.0-3347/tars/phoenix-4.4.0.2.3.4.0-3347.tar.gz</phoenix.tar> - <phoenix.folder>phoenix-4.4.0.2.3.4.0-3347</phoenix.folder> + <phoenix.tar>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.0.3/tars/phoenix-4.7.0.2.6.0.3-8.tar.gz</phoenix.tar> + <phoenix.folder>phoenix-4.7.0.2.6.0.3-8</phoenix.folder> <resmonitor.install.dir> /usr/lib/python2.6/site-packages/resource_monitoring </resmonitor.install.dir> http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java index ecb59ac..b570d19 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/metrics/timeline/MetricsRequestHelper.java @@ -107,7 +107,6 @@ public class MetricsRequestHelper { ", host = " + metric.getHostName() + ", app = " + metric.getAppId() + ", instance = " + metric.getInstanceId() + - ", time = " + metric.getTimestamp() + ", startTime = " + new Date(metric.getStartTime())); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/smoketest_metrics.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/smoketest_metrics.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/smoketest_metrics.json.j2 index 2ee0efa..38c99eb 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/smoketest_metrics.json.j2 +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/package/templates/smoketest_metrics.json.j2 @@ -4,7 +4,6 @@ "metricname": "AMBARI_METRICS.SmokeTest.FakeMetric", "appid": "amssmoketestfake", "hostname": "{{hostname}}", - "timestamp": {{current_time}}, "starttime": {{current_time}}, "metrics": { "{{current_time}}": {{random1}}, http://git-wip-us.apache.org/repos/asf/ambari/blob/84534df7/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java index fbc0b56..09b2c39 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/metrics/timeline/cache/TimelineMetricCacheSizingTest.java @@ -41,7 +41,6 @@ public class TimelineMetricCacheSizingTest { metric.setAppId("KAFKA_BROKER"); metric.setInstanceId("NULL"); metric.setHostName("my.privatehostname.of.average.length"); - metric.setTimestamp(System.currentTimeMillis()); metric.setStartTime(System.currentTimeMillis()); metric.setType("LONG");