This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
commit c5c9d03328bd1dbf7ad86434d209709592b0ac27 Author: Aravindan Vijayan <[email protected]> AuthorDate: Fri Mar 16 11:08:56 2018 -0700 AMBARI-23250 : Fix deployment issues in AMS perf branch. --- .../ambari-metrics-timelineservice/pom.xml | 64 ++++++++++++++++++---- .../timeline/uuid/HashBasedUuidGenStrategy.java | 60 ++++++++++---------- .../src/main/resources/metrics_def/HOST.dat | 6 ++ .../webapp/TestTimelineWebServices.java | 6 -- 4 files changed, 90 insertions(+), 46 deletions(-) diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml index 6a6dc3e..13076f5 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml +++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml @@ -336,6 +336,22 @@ <groupId>net.sourceforge.findbugs</groupId> <artifactId>annotations</artifactId> </exclusion> + <exclusion> + <artifactId>jersey-server</artifactId> + <groupId>org.glassfish.jersey.core</groupId> + </exclusion> + <exclusion> + <artifactId>jersey-common</artifactId> + <groupId>org.glassfish.jersey.core</groupId> + </exclusion> + <exclusion> + <artifactId>jersey-container-servlet-core</artifactId> + <groupId>org.glassfish.jersey.containers</groupId> + </exclusion> + <exclusion> + <artifactId>javax.ws.rs-api</artifactId> + <groupId>javax.ws.rs</groupId> + </exclusion> </exclusions> </dependency> @@ -450,6 +466,16 @@ <artifactId>jersey-server</artifactId> <version>1.11</version> </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + <version>1.11</version> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-client</artifactId> + <version>1.11</version> + </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <dependency> <groupId>org.apache.hadoop</groupId> @@ -481,16 +507,6 @@ <version>1.1</version> </dependency> <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-core</artifactId> - <version>1.11</version> - </dependency> - <dependency> - <groupId>com.sun.jersey</groupId> - <artifactId>jersey-client</artifactId> - <version>1.11</version> - </dependency> - <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> <version>4.5.2</version> @@ -653,6 +669,24 @@ <groupId>org.apache.phoenix</groupId> <artifactId>phoenix-core</artifactId> <type>test-jar</type> + <exclusions> + <exclusion> + <artifactId>jersey-server</artifactId> + <groupId>org.glassfish.jersey.core</groupId> + </exclusion> + <exclusion> + <artifactId>jersey-common</artifactId> + <groupId>org.glassfish.jersey.core</groupId> + </exclusion> + <exclusion> + <artifactId>jersey-container-servlet-core</artifactId> + <groupId>org.glassfish.jersey.containers</groupId> + </exclusion> + <exclusion> + <artifactId>javax.ws.rs-api</artifactId> + <groupId>javax.ws.rs</groupId> + </exclusion> + </exclusions> <version>${phoenix.version}</version> <scope>test</scope> </dependency> @@ -661,6 +695,16 @@ <artifactId>hbase-it</artifactId> <version>${hbase.version}</version> <scope>test</scope> + <exclusions> + <exclusion> + <artifactId>jersey-common</artifactId> + <groupId>org.glassfish.jersey.core</groupId> + </exclusion> + <exclusion> + <artifactId>javax.ws.rs-api</artifactId> + <groupId>javax.ws.rs</groupId> + </exclusion> + </exclusions> <classifier>tests</classifier> </dependency> <dependency> diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/HashBasedUuidGenStrategy.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/HashBasedUuidGenStrategy.java index 3acf656..16d1bf2 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/HashBasedUuidGenStrategy.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/uuid/HashBasedUuidGenStrategy.java @@ -37,7 +37,6 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { */ @Override public byte[] computeUuid(TimelineClusterMetric timelineClusterMetric, int maxLength) { - int metricNameUuidLength = 12; String metricName = timelineClusterMetric.getMetricName(); @@ -51,16 +50,10 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { */ StringBuilder splitSums = new StringBuilder(); if (splits.length > 0) { - for (int s = 0; s < splits.length; s++) { + for (String split : splits) { int asciiSum = 0; - if ( s < splits.length -1) { - for (int i = 0; i < splits[s].length(); i++) { - asciiSum += (int) splits[s].charAt(i); // Get Ascii Sum. - } - } else { - for (int i = 0; i < splits[s].length(); i++) { - asciiSum += ((i+1) * (int) splits[s].charAt(i)); //weighted sum for last split. - } + for (int i = 0; i < split.length(); i++) { + asciiSum += ((i + 1) * (int) split.charAt(i)); //weighted sum for split. } splitSums.append(asciiSum); //Append the sum to the array of sums. } @@ -69,9 +62,7 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { //Compute a unique metric seed for the stemmed metric name String stemmedMetric = stem(metricName); long metricSeed = 100123456789L; - for (int i = 0; i < stemmedMetric.length(); i++) { - metricSeed += stemmedMetric.charAt(i); - } + metricSeed += computeWeightedNumericalAsciiSum(stemmedMetric); //Reverse the computed seed to get a metric UUID portion which is used optionally. byte[] metricUuidPortion = StringUtils.reverse(String.valueOf(metricSeed)).getBytes(); @@ -80,7 +71,9 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { //If splitSums length > required metric UUID length, use only the required length suffix substring of the splitSums as metric UUID. if (splitLength > metricNameUuidLength) { - metricUuidPortion = ArrayUtils.subarray(splitSumString.getBytes(), splitLength - metricNameUuidLength, splitLength); + int pad = (int)(0.25 * splitLength); + metricUuidPortion = ArrayUtils.addAll(ArrayUtils.subarray(splitSumString.getBytes(), splitLength - metricNameUuidLength + pad, splitLength) + , ArrayUtils.subarray(metricUuidPortion, 0, pad)); } else { //If splitSums is not enough for required metric UUID length, pad with the metric uuid portion. int pad = metricNameUuidLength - splitLength; @@ -94,7 +87,7 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { String appId = timelineClusterMetric.getAppId(); int appidSeed = 11; for (int i = 0; i < appId.length(); i++) { - appidSeed += appId.charAt(i); + appidSeed += ((i+1) * appId.charAt(i)); } String appIdSeedStr = String.valueOf(appidSeed); byte[] appUuidPortion = ArrayUtils.subarray(appIdSeedStr.getBytes(), appIdSeedStr.length() - 2, appIdSeedStr.length()); @@ -105,7 +98,7 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { if (StringUtils.isNotEmpty(instanceId)) { int instanceIdSeed = 1489; for (int i = 0; i < appId.length(); i++) { - instanceIdSeed += appId.charAt(i); + instanceIdSeed += ((i+1)* appId.charAt(i)); } buffer.putInt(instanceIdSeed); ArrayUtils.subarray(buffer.array(), 2, 4); @@ -126,11 +119,12 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { private String[] getIndidivualSplits(String metricName) { List<String> tokens = new ArrayList<>(); String[] splits = new String[0]; - if (metricName.contains("\\.")) { + if (metricName.contains(".")) { splits = metricName.split("\\."); for (String split : splits) { if (split.contains("_")) { - tokens.addAll(Arrays.asList(split.split("_"))); + String[] subSplits = split.split("\\_"); + tokens.addAll(Arrays.asList(subSplits)); } else { tokens.add(split); } @@ -176,31 +170,37 @@ public class HashBasedUuidGenStrategy implements MetricUuidGenStrategy { if (StringUtils.isEmpty(value)) { return null; } + + int customAsciiSum = 1489 + (int) computeWeightedNumericalAsciiSum(value); //seed = 1489 + + String customAsciiSumStr = String.valueOf(customAsciiSum); + if (customAsciiSumStr.length() < maxLength) { + return null; + } else { + return customAsciiSumStr.substring(customAsciiSumStr.length() - maxLength, customAsciiSumStr.length()).getBytes(); + } + } + + private long computeWeightedNumericalAsciiSum(String value) { int len = value.length(); - int numericValue = 0; - int seed = 1489; + long numericValue = 0; + int sum = 0; for (int i = 0; i < len; i++) { int ascii = value.charAt(i); if (48 <= ascii && ascii <= 57) { numericValue += numericValue * 10 + (ascii - 48); } else { if (numericValue > 0) { - seed += numericValue; + sum += numericValue; numericValue = 0; } - seed+= value.charAt(i); + sum += value.charAt(i); } } if (numericValue != 0) { - seed+=numericValue; - } - - String seedStr = String.valueOf(seed); - if (seedStr.length() < maxLength) { - return null; - } else { - return seedStr.substring(seedStr.length() - maxLength, seedStr.length()).getBytes(); + sum +=numericValue; } + return sum; } } \ No newline at end of file diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/HOST.dat b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/HOST.dat index 3758140..6a7034b 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/HOST.dat +++ b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/metrics_def/HOST.dat @@ -53,3 +53,9 @@ write_bps write_bytes write_count write_time +sdisk_dm-12_write_count +sdisk_dm-21_write_count +sdisk_dm-40_write_bytes +sdisk_dm-22_write_bytes +sdisk_dm-26_write_time +sdisk_dm-17_write_time diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java index 83e2a27..cd20470 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java +++ b/ambari-metrics/ambari-metrics-timelineservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/TestTimelineWebServices.java @@ -23,14 +23,8 @@ import static org.junit.Assert.assertEquals; import javax.ws.rs.core.MediaType; import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; -import org.apache.hadoop.yarn.api.records.timeline.TimelineEvents; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TestTimelineMetricStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.junit.Test; -- To stop receiving notification emails like this one, please contact [email protected].
