http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java deleted file mode 100644 index 7974a5f..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/LoadRunner.java +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator; - -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data.AppID; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data.ApplicationInstance; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data.HostMetricsGenerator; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data.MetricsGeneratorConfigurer; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.net.MetricsSender; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.net.RestMetricsSender; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.net.StdOutMetricsSender; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util.TimeStampProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.List; -import java.util.concurrent.*; - -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data.AppID.MASTER_APPS; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data.AppID.SLAVE_APPS; - -/** - * - */ -public class LoadRunner { - private final static Logger LOG = LoggerFactory.getLogger(LoadRunner.class); - - private final ScheduledExecutorService timer; - private final ExecutorService workersPool; - private final Collection<Callable<String>> workers; - private final long startTime = new Date().getTime(); - private final int collectIntervalMillis; - private final int sendIntervalMillis; - - public LoadRunner(String hostName, - int threadCount, - String metricsHostName, - int collectIntervalMillis, - int sendIntervalMillis, - boolean createMaster) { - this.collectIntervalMillis = collectIntervalMillis; - this.workersPool = Executors.newFixedThreadPool(threadCount); - this.timer = Executors.newScheduledThreadPool(1); - this.sendIntervalMillis = sendIntervalMillis; - - workers = prepareWorkers(hostName, threadCount, metricsHostName, createMaster); - } - - private Collection<Callable<String>> prepareWorkers(String hostName, - int threadCount, - String metricsHost, - Boolean createMaster) { - Collection<Callable<String>> senderWorkers = - new ArrayList<Callable<String>>(threadCount); - - int startIndex = 0; - if (createMaster) { - String simHost = hostName + ".0"; - addMetricsWorkers(senderWorkers, simHost, metricsHost, MASTER_APPS); - startIndex++; - } - - for (int i = startIndex; i < threadCount; i++) { - String simHost = hostName + "." + i; - addMetricsWorkers(senderWorkers, simHost, metricsHost, SLAVE_APPS); - } - - return senderWorkers; - } - - private void addMetricsWorkers(Collection<Callable<String>> senderWorkers, - String specificHostName, - String metricsHostName, - AppID[] apps) { - for (AppID app : apps) { - HostMetricsGenerator metricsGenerator = - createApplicationMetrics(specificHostName, app); - MetricsSender sender = new RestMetricsSender(metricsHostName); - senderWorkers.add(new MetricsSenderWorker(sender, metricsGenerator)); - } - } - - private HostMetricsGenerator createApplicationMetrics(String simHost, AppID host) { - ApplicationInstance appInstance = new ApplicationInstance(simHost, host, ""); - TimeStampProvider timeStampProvider = new TimeStampProvider(startTime, - collectIntervalMillis, sendIntervalMillis); - - return MetricsGeneratorConfigurer - .createMetricsForHost(appInstance, timeStampProvider); - } - - public void start() { - timer.scheduleAtFixedRate(new Runnable() { - @Override - public void run() { - try { - runOnce(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - }, 0, sendIntervalMillis, TimeUnit.MILLISECONDS); - } - - public void runOnce() throws InterruptedException { - List<Future<String>> futures = workersPool.invokeAll(workers, - sendIntervalMillis / 2, - TimeUnit.MILLISECONDS); - int done = 0; - - // TODO: correctly count the failed tasks - for (Future<String> future : futures) { - done += future.isDone() ? 1 : 0; - } - - LOG.info("Finished successfully " + done + " tasks "); - } - - public void shutdown() { - timer.shutdownNow(); - workersPool.shutdownNow(); - } - - public static void main(String[] args) { - LoadRunner runner = - new LoadRunner("local", 2, "metrics", 10000, 20000, false); - - runner.start(); - } - -}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java deleted file mode 100644 index a0c1bd2..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsLoadSimulator.java +++ /dev/null @@ -1,120 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -/** - * Sample Usage: - * <pre> - * $ java -cp "dependency/*":LoadSimulator-1.0-SNAPSHOT.jar \ - * org.apache.ambari.metrics.MetricsLoadSimulator \ - * -h "bartosz.laptop" -n 2 -m "162.216.148.45" -c 10000 -s 30000</pre> - */ -public class MetricsLoadSimulator { - private final static Logger LOG = LoggerFactory.getLogger(MetricsLoadSimulator - .class); - - public static void main(String[] args) throws IOException, InterruptedException { - Map<String, String> mapArgs = parseArgs(args); - - LoadRunner loadRunner = new LoadRunner( - mapArgs.get("hostName"), - Integer.valueOf(mapArgs.get("numberOfHosts")), - mapArgs.get("metricsHostName"), - Integer.valueOf(mapArgs.get("collectInterval")), - Integer.valueOf(mapArgs.get("sendInterval")), - Boolean.valueOf(mapArgs.get("master")) - ); - - loadRunner.start(); - } - - private static Map<String, String> parseArgs(String[] args) { - Map<String, String> mapProps = new HashMap<String, String>(); - mapProps.put("hostName", "host"); - mapProps.put("numberOfHosts", "20"); - mapProps.put("trafficType", "burst"); - mapProps.put("metricsHostName", "localhost"); - mapProps.put("collectInterval", "10000"); - mapProps.put("sendInterval", "60000"); - - if (args.length == 0) { - printUsage(); - throw new RuntimeException("Unexpected argument, See usage message."); - } else { - for (int i = 0; i < args.length; i += 2) { - String arg = args[i]; - if (arg.equals("-h")) { - mapProps.put("hostName", args[i + 1]); - } else if (arg.equals("-n")) { - mapProps.put("numberOfHosts", args[i + 1]); - } else if (arg.equals("-t")) { - mapProps.put("trafficType", args[i + 1]); - } else if (arg.equals("-m")) { - mapProps.put("metricsHostName", args[i + 1]); - } else if (arg.equals("-c")) { - mapProps.put("collectInterval", args[i + 1]); - } else if (arg.equals("-s")) { - mapProps.put("sendInterval", args[i + 1]); - } else if (arg.equals("-M")) { - mapProps.put("master", args[i + 1]); - } else if (arg.equals("-d")) { - // a dummy switch - it says that we agree with defaults - } else { - printUsage(); - throw new RuntimeException("Unexpected argument, See usage message."); - } - } - } - - LOG.info("Recognized options: baseHostName={} hosts#={} trafficMode={} " + - "metricsHostName={} collectIntervalMillis={} sendIntervalMillis={} " + - "simulateMaster={}", - mapProps.get("hostName"), - Integer.valueOf(mapProps.get("numberOfHosts")), - mapProps.get("trafficType"), - mapProps.get("metricsHostName"), - Integer.valueOf(mapProps.get("collectInterval")), - Integer.valueOf(mapProps.get("sendInterval")), - Boolean.valueOf(mapProps.get("master")) - ); - - return mapProps; - } - - public static void printUsage() { - System.err.println("Usage: java MetricsLoadSimulator [OPTIONS]"); - System.err.println("Options: "); - System.err.println("[-h hostName] [-n numberOfHosts] " - + "[-t trafficMode {burst, staggered}] [-m metricsHostName] " - + "[-c collectIntervalMillis {10 sec}] [-s sendIntervalMillis {60 sec}]" - + "[-M simulateMaster {true, false}] "); - System.err.println(); - System.err.println("When you select a master, then one simulated host will play"); - System.err.println("a role of a master, and the rest will be slaves. Otherwise"); - System.err.println("all simulation threads (single thread is for single host)"); - System.err.println("will be slave hosts"); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsSenderWorker.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsSenderWorker.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsSenderWorker.java deleted file mode 100644 index c027933..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/MetricsSenderWorker.java +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator; - - -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data.AppMetrics; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data.HostMetricsGenerator; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.net.MetricsSender; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.net.RestMetricsSender; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util.Json; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.concurrent.Callable; - -/** - */ -public class MetricsSenderWorker implements Callable<String> { - private final static Logger LOG = LoggerFactory.getLogger(RestMetricsSender.class); - - MetricsSender sender; - HostMetricsGenerator hmg; - - public MetricsSenderWorker(MetricsSender sender, HostMetricsGenerator metricsGenerator) { - this.sender = sender; - hmg = metricsGenerator; - } - - @Override - public String call() throws Exception { - AppMetrics hostMetrics = hmg.createMetrics(); - - try { - String request = new Json().serialize(hostMetrics); - String response = sender.pushMetrics(request); - - return response; - } catch (IOException e) { - LOG.error("Error while pushing metrics: ", e); - throw e; - } - - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java deleted file mode 100644 index 4f58dc5..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppID.java +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data; - -public enum AppID { - HOST("HOST"), - NAMENODE("namenode"), - RESOURCEMANAGER("resourcemanager"), - DATANODE("datanode"), - NODEMANAGER("nodemanager"), - MASTER_HBASE("hbase"), - SLAVE_HBASE("hbase"); - - public static final AppID[] MASTER_APPS = {HOST, NAMENODE, RESOURCEMANAGER, MASTER_HBASE}; - public static final AppID[] SLAVE_APPS = {HOST, DATANODE, NODEMANAGER, SLAVE_HBASE}; - - private String id; - - private AppID(String id) { - this.id = id; - } - - public String getId() { - return id; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppMetrics.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppMetrics.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppMetrics.java deleted file mode 100644 index d9cec2b..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/AppMetrics.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data; - -import java.util.ArrayList; -import java.util.Collection; - -/** - * AppMetrics is a class that helps to create properly initialized metrics for - * current app. It also holds the - * metrics and can be serialized to json. - */ -public class AppMetrics { - - private final Collection<Metric> metrics = new ArrayList<Metric>(); - private final transient ApplicationInstance applicationId; - private final transient long startTime; - - public AppMetrics(ApplicationInstance applicationId, long startTime) { - this.applicationId = applicationId; - this.startTime = startTime; - } - - public Metric createMetric(String metricName) { - return new Metric(applicationId, metricName, startTime); - } - - public void addMetric(Metric metric) { - metrics.add(metric); - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/ApplicationInstance.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/ApplicationInstance.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/ApplicationInstance.java deleted file mode 100644 index d99ecc9..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/ApplicationInstance.java +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.data; - -/** - * AppId is a helper class that encapsulates the common part of metrics message. - * It contains hostName, appId and instanceId. It is immutable, - * and it can not hold null values. - */ -public final class ApplicationInstance { - - private final transient String hostName; - private final transient AppID appId; - private final transient String instanceId; - - /** - * @param hostname - * @param appId - * @param instanceId - */ - public ApplicationInstance(String hostname, AppID appId, String instanceId) { - if (hostname == null || appId == null || instanceId == null) - throw new IllegalArgumentException("ApplicationInstance can not be " + - "instantiated with null values"); - - this.hostName = hostname; - this.appId = appId; - this.instanceId = instanceId; - } - - public String getInstanceId() { - return instanceId; - } - - public AppID getAppId() { - return appId; - } - - public String getHostName() { - return hostName; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/HostMetricsGenerator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/HostMetricsGenerator.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/HostMetricsGenerator.java deleted file mode 100644 index 61a6624..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/HostMetricsGenerator.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data; - - -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util.RandomMetricsProvider; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util.TimeStampProvider; - -import java.util.HashMap; -import java.util.Map; - -/** - */ -public class HostMetricsGenerator { - - private Map<String, RandomMetricsProvider> metricDataProviders = new HashMap<String, RandomMetricsProvider>(); - private final TimeStampProvider tsp; - private final ApplicationInstance id; - - public HostMetricsGenerator(ApplicationInstance id, - TimeStampProvider timeStamps, - Map<String, RandomMetricsProvider> metricDataProviders) { - this.id = id; - this.tsp = timeStamps; - this.metricDataProviders = metricDataProviders; - } - - public AppMetrics createMetrics() { - long[] timestamps = tsp.timestampsForNextInterval(); - AppMetrics appMetrics = new AppMetrics(id, timestamps[0]); - - for (Map.Entry<String, RandomMetricsProvider> entry : metricDataProviders.entrySet()) { - String metricName = entry.getKey(); - RandomMetricsProvider metricData = entry.getValue(); - - Metric metric = appMetrics.createMetric(metricName); - for (long timestamp : timestamps) { - metric.putMetric(timestamp, String.valueOf(metricData.next())); - } - appMetrics.addMetric(metric); - } - - return appMetrics; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/Metric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/Metric.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/Metric.java deleted file mode 100644 index f274263..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/Metric.java +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data; - -import java.util.LinkedHashMap; -import java.util.Map; - -public class Metric { - - private String instanceid; - private String hostname; - private Map<String, String> metrics = new LinkedHashMap<String, String>(); - private String starttime; - private String appid; - private String metricname; - - // i don't like this ctor, but it has to be public for json deserialization - public Metric() { - } - - public Metric(ApplicationInstance app, String metricName, long startTime) { - this.hostname = app.getHostName(); - this.appid = app.getAppId().getId(); - this.instanceid = app.getInstanceId(); - this.metricname = metricName; - this.starttime = Long.toString(startTime); - } - - public void putMetric(long timestamp, String value) { - metrics.put(Long.toString(timestamp), value); - } - - public String getInstanceid() { - return instanceid; - } - - public String getHostname() { - return hostname; - } - - public Map<String, String> getMetrics() { - return metrics; - } - - public String getStarttime() { - return starttime; - } - - public String getAppid() { - return appid; - } - - public String getMetricname() { - return metricname; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/MetricsGeneratorConfigurer.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/MetricsGeneratorConfigurer.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/MetricsGeneratorConfigurer.java deleted file mode 100644 index b3401b2..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/data/MetricsGeneratorConfigurer.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.data; - - -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util.RandomMetricsProvider; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util.TimeStampProvider; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.HashMap; -import java.util.Map; - -/** - * MetricsGeneratorConfigurer is a factory that reads metrics definition from a file, - * and returns an Single HostMetricsGenerator. Check createMetricsForHost method - * for details. - */ -public class MetricsGeneratorConfigurer { - - private final static Logger LOG = LoggerFactory.getLogger - (MetricsGeneratorConfigurer.class); - - /** - * Creates HostMetricsGenerator configured with metric names loaded from file. - * - * @param id ApplicationInstance descriptor, will be used to create - * HostMetricsGenerator, cannot be null - * @param timeStamps configured TimeStampProvider that can provide next - * timestamp, cannot be null - * @return HostMetricsGenerator with given ApplicationInstance id and configured - * mapping of - * metric names to data providers - */ - public static HostMetricsGenerator createMetricsForHost( - ApplicationInstance id, - TimeStampProvider timeStamps) { - return new HostMetricsGenerator(id, timeStamps, readMetrics(id.getAppId())); - } - - private static Map<String, RandomMetricsProvider> readMetrics(AppID type) { - InputStream input = null; - Map<String, RandomMetricsProvider> metrics = - new HashMap<String, RandomMetricsProvider>(); - String fileName = "metrics_def/" + type.toString() + ".dat"; - - try { - LOG.info("Loading " + fileName); - - input = MetricsGeneratorConfigurer.class.getClassLoader() - .getResourceAsStream(fileName); - - BufferedReader reader = new BufferedReader(new InputStreamReader(input)); - - String line; - while ((line = reader.readLine()) != null) { - metrics.put(line.trim(), new RandomMetricsProvider(100, 200)); - } - - } catch (IOException e) { - LOG.error("Cannot read file " + fileName + " for appID " + type.toString(), e); - } finally { - if (input != null) { - try { - input.close(); - } catch (IOException ex) { - // intentionally left blank, here we cannot do anything - } - } - } - - return metrics; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/MetricsSender.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/MetricsSender.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/MetricsSender.java deleted file mode 100644 index 35c0fc3..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/MetricsSender.java +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net; - -/** - * MetricSender should provides a simple way of pushing metrics to some service. - */ -public interface MetricsSender { - /** - * Push metrics to the metric service (e.g. a metrics storage system). - * - * @param payload the payload to be sent to metrics service - * @return response message either acknowledgement or error - */ - String pushMetrics(String payload); -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java deleted file mode 100644 index 5130ae3..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/RestMetricsSender.java +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net; - -import com.google.common.base.Stopwatch; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.ProtocolException; - -/** - * Implements MetricsSender and provides a way of pushing metrics to application metrics history service using REST - * endpoint. - */ -public class RestMetricsSender implements MetricsSender { - private final static Logger LOG = LoggerFactory.getLogger(RestMetricsSender.class); - - private final static String COLLECTOR_URL = "http://%s:8188/ws/v1/timeline/metrics";; - private final String collectorServiceAddress; - - /** - * Creates unconnected RestMetricsSender with endpoint configured as - * http://${metricsHost}:8188/ws/v1/timeline/metrics, - * where ${metricsHost} is specified by metricHost param. - * - * @param metricsHost the hostname that will be used to access application metrics history service. - */ - public RestMetricsSender(String metricsHost) { - collectorServiceAddress = String.format(COLLECTOR_URL, metricsHost); - } - - /** - * Push metrics to the REST endpoint. Connection is always open and closed on every call. - * - * @param payload the payload with metrics to be sent to metrics service - * @return response message either acknowledgement or error, empty on exception - */ - @Override - public String pushMetrics(String payload) { - String responseString = ""; - UrlService svc = null; - Stopwatch timer = new Stopwatch().start(); - - try { - LOG.info("server: {}", collectorServiceAddress); - - svc = getConnectedUrlService(); - responseString = svc.send(payload); - - timer.stop(); - LOG.info("http response time: " + timer.elapsedMillis() + " ms"); - - if (responseString.length() > 0) { - LOG.debug("POST response from server: " + responseString); - } - } catch (MalformedURLException e) { - LOG.error("", e); - } catch (ProtocolException e) { - LOG.error("", e); - } catch (IOException e) { - LOG.error("", e); - } finally { - if (svc != null) { - svc.disconnect(); - } - } - - return responseString; - } - - /** - * Relaxed to protected for testing. - */ - protected UrlService getConnectedUrlService() throws IOException { - return UrlService.newConnection(collectorServiceAddress); - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/StdOutMetricsSender.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/StdOutMetricsSender.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/StdOutMetricsSender.java deleted file mode 100644 index aeb4ca8..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/StdOutMetricsSender.java +++ /dev/null @@ -1,56 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.net; - -import java.io.PrintStream; - -/** - * StdOutMetricsSender dumps metrics to defined PrintStream out. It is useful for testing. - */ -public class StdOutMetricsSender implements MetricsSender { - public final PrintStream out; - private String metricsHostName; - - /** - * Creates new StdOutMetricsSender with specified hostname (only used in messages) and sends output to System.out - * - * @param metricsHostName a name used in printed messages - */ - public StdOutMetricsSender(String metricsHostName) { - this(metricsHostName, System.out); - } - - /** - * Creates new StdOutMetricsSender with specified hostname (only used in messages) and PrintStream which is used as - * an output. - * - * @param metricsHostName a name used in printed messages - * @param out PrintStream that the Sender will write to, can be System.out - */ - public StdOutMetricsSender(String metricsHostName, PrintStream out) { - this.metricsHostName = metricsHostName; - this.out = out; - } - - @Override - public String pushMetrics(String payload) { - out.println("Sending to " + metricsHostName + ": " + payload); - - return "OK"; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/UrlService.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/UrlService.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/UrlService.java deleted file mode 100644 index 7402438..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/net/UrlService.java +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.net; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.net.HttpURLConnection; -import java.net.URL; - -public class UrlService { - - public static final int CONNECT_TIMEOUT = 20000; - public static final int READ_TIMEOUT = 20000; - private final String address; - private HttpURLConnection conn; - - private UrlService(String address) { - this.address = address; - } - - /** - * Returns a new UrlService connected to specified address. - * - * @param address - * @return - * @throws IOException - */ - public static UrlService newConnection(String address) throws IOException { - UrlService svc = new UrlService(address); - svc.connect(); - - return svc; - } - - public HttpURLConnection connect() throws IOException { - URL url = new URL(address); - conn = (HttpURLConnection) url.openConnection(); - - //TODO: make timeouts configurable - conn.setConnectTimeout(CONNECT_TIMEOUT); - conn.setReadTimeout(READ_TIMEOUT); - conn.setDoInput(true); - conn.setDoOutput(true); - conn.setRequestMethod("POST"); - conn.setRequestProperty("Content-Type", "application/json"); - conn.setRequestProperty("Accept", "*/*"); - - return conn; - } - - public String send(String payload) throws IOException { - if (conn == null) - throw new IllegalStateException("Cannot use unconnected UrlService"); - write(payload); - - return read(); - } - - private String read() throws IOException { - StringBuilder response = new StringBuilder(); - - BufferedReader br = new BufferedReader(new InputStreamReader( - conn.getInputStream())); - String line = null; - while ((line = br.readLine()) != null) { - response.append(line); - } - br.close(); - - return response.toString(); - } - - private void write(String payload) throws IOException { - OutputStreamWriter writer = new OutputStreamWriter(conn.getOutputStream(), - "UTF-8"); - writer.write(payload); - writer.close(); - } - - public void disconnect() { - conn.disconnect(); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/Json.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/Json.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/Json.java deleted file mode 100644 index 61a3903..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/Json.java +++ /dev/null @@ -1,63 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util; - -import org.codehaus.jackson.annotate.JsonAutoDetect; -import org.codehaus.jackson.annotate.JsonMethod; -import org.codehaus.jackson.map.ObjectMapper; -import org.codehaus.jackson.map.SerializationConfig; - -import java.io.IOException; - -/** - * Small wrapper that configures the ObjectMapper with some defaults. - */ -public class Json { - private ObjectMapper myObjectMapper; - - /** - * Creates default Json ObjectMapper that maps fields. - */ - public Json() { - this(false); - } - - /** - * Creates a Json ObjectMapper that maps fields and optionally pretty prints the - * serialized objects. - * - * @param pretty a flag - if true the output will be pretty printed. - */ - public Json(boolean pretty) { - myObjectMapper = new ObjectMapper(); - myObjectMapper.setVisibility(JsonMethod.FIELD, JsonAutoDetect.Visibility.ANY); - if (pretty) { - myObjectMapper.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); - } - } - - public String serialize(Object o) throws IOException { - return myObjectMapper.writeValueAsString(o); - } - - public <T> T deserialize(String content, Class<T> paramClass) throws IOException { - return myObjectMapper.readValue(content, paramClass); - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/RandomMetricsProvider.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/RandomMetricsProvider.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/RandomMetricsProvider.java deleted file mode 100644 index 7910711..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/RandomMetricsProvider.java +++ /dev/null @@ -1,40 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util; - -import java.util.Random; - -/** - */ -public class RandomMetricsProvider { - - private double min; - private double max; - private Random rnd; - - public RandomMetricsProvider(double min, double max) { - this.min = min; - this.max = max; - this.rnd = new Random(); - } - - public double next() { - return rnd.nextDouble() * (max - min) + min; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TimeStampProvider.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TimeStampProvider.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TimeStampProvider.java deleted file mode 100644 index ad7ec86..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/util/TimeStampProvider.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .loadsimulator.util; - -/** - */ -public class TimeStampProvider { - private int timeStep; - private long currentTime; - private int sendInterval; - - public TimeStampProvider(long startTime, int timeStep, int sendInterval) { - this.timeStep = timeStep; - this.currentTime = startTime - timeStep; - this.sendInterval = sendInterval; - } - - public long next() { - return currentTime += timeStep; - } - - public long[] timestampsForNextInterval() { - return timestampsForInterval(sendInterval); - } - - private long[] timestampsForInterval(int sendInterval) { - int steps = sendInterval / timeStep; - long[] timestamps = new long[steps]; - - for (int i = 0; i < timestamps.length; i++) { - timestamps[i] = next(); - } - - return timestamps; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java deleted file mode 100644 index a123e57..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/AbstractTimelineAggregator.java +++ /dev/null @@ -1,229 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; - -import java.io.File; -import java.io.IOException; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.Date; - -import static java.util.concurrent.TimeUnit.SECONDS; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.AGGREGATOR_CHECKPOINT_DELAY; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline.TimelineMetricConfiguration.RESULTSET_FETCH_SIZE; - -public abstract class AbstractTimelineAggregator implements Runnable { - protected final PhoenixHBaseAccessor hBaseAccessor; - private final Log LOG; - protected final long checkpointDelayMillis; - protected final Integer resultsetFetchSize; - protected Configuration metricsConf; - - public AbstractTimelineAggregator(PhoenixHBaseAccessor hBaseAccessor, - Configuration metricsConf) { - this.hBaseAccessor = hBaseAccessor; - this.metricsConf = metricsConf; - this.checkpointDelayMillis = SECONDS.toMillis( - metricsConf.getInt(AGGREGATOR_CHECKPOINT_DELAY, 120)); - this.resultsetFetchSize = metricsConf.getInt(RESULTSET_FETCH_SIZE, 2000); - this.LOG = LogFactory.getLog(this.getClass()); - } - - @Override - public void run() { - LOG.info("Started Timeline aggregator thread @ " + new Date()); - Long SLEEP_INTERVAL = getSleepIntervalMillis(); - - while (true) { - long currentTime = System.currentTimeMillis(); - long lastCheckPointTime = -1; - - try { - lastCheckPointTime = readCheckPoint(); - if (isLastCheckPointTooOld(lastCheckPointTime)) { - LOG.warn("Last Checkpoint is too old, discarding last checkpoint. " + - "lastCheckPointTime = " + lastCheckPointTime); - lastCheckPointTime = -1; - } - if (lastCheckPointTime == -1) { - // Assuming first run, save checkpoint and sleep. - // Set checkpoint to 2 minutes in the past to allow the - // agents/collectors to catch up - saveCheckPoint(currentTime - checkpointDelayMillis); - } - } catch (IOException io) { - LOG.warn("Unable to write last checkpoint time. Resuming sleep.", io); - } - long sleepTime = SLEEP_INTERVAL; - - if (lastCheckPointTime != -1) { - LOG.info("Last check point time: " + lastCheckPointTime + ", lagBy: " - + ((System.currentTimeMillis() - lastCheckPointTime) / 1000) - + " seconds."); - - long startTime = System.currentTimeMillis(); - boolean success = doWork(lastCheckPointTime, - lastCheckPointTime + SLEEP_INTERVAL); - long executionTime = System.currentTimeMillis() - startTime; - long delta = SLEEP_INTERVAL - executionTime; - - if (delta > 0) { - // Sleep for (configured sleep - time to execute task) - sleepTime = delta; - } else { - // No sleep because last run took too long to execute - LOG.info("Aggregator execution took too long, " + - "cancelling sleep. executionTime = " + executionTime); - sleepTime = 1; - } - - LOG.debug("Aggregator sleep interval = " + sleepTime); - - if (success) { - try { - saveCheckPoint(lastCheckPointTime + SLEEP_INTERVAL); - } catch (IOException io) { - LOG.warn("Error saving checkpoint, restarting aggregation at " + - "previous checkpoint."); - } - } - } - - try { - Thread.sleep(sleepTime); - } catch (InterruptedException e) { - LOG.info("Sleep interrupted, continuing with aggregation."); - } - } - } - - private boolean isLastCheckPointTooOld(long checkpoint) { - return checkpoint != -1 && - ((System.currentTimeMillis() - checkpoint) > - getCheckpointCutOffIntervalMillis()); - } - - private long readCheckPoint() { - try { - File checkpoint = new File(getCheckpointLocation()); - if (checkpoint.exists()) { - String contents = FileUtils.readFileToString(checkpoint); - if (contents != null && !contents.isEmpty()) { - return Long.parseLong(contents); - } - } - } catch (IOException io) { - LOG.debug(io); - } - return -1; - } - - private void saveCheckPoint(long checkpointTime) throws IOException { - File checkpoint = new File(getCheckpointLocation()); - if (!checkpoint.exists()) { - boolean done = checkpoint.createNewFile(); - if (!done) { - throw new IOException("Could not create checkpoint at location, " + - getCheckpointLocation()); - } - } - FileUtils.writeStringToFile(checkpoint, String.valueOf(checkpointTime)); - } - - /** - * Read metrics written during the time interval and save the sum and total - * in the aggregate table. - * - * @param startTime Sample start time - * @param endTime Sample end time - */ - protected boolean doWork(long startTime, long endTime) { - LOG.info("Start aggregation cycle @ " + new Date() + ", " + - "startTime = " + new Date(startTime) + ", endTime = " + new Date(endTime)); - - boolean success = true; - PhoenixTransactSQL.Condition condition = - prepareMetricQueryCondition(startTime, endTime); - - Connection conn = null; - PreparedStatement stmt = null; - - try { - conn = hBaseAccessor.getConnection(); - stmt = PhoenixTransactSQL.prepareGetMetricsSqlStmt(conn, condition); - - LOG.debug("Query issued @: " + new Date()); - ResultSet rs = stmt.executeQuery(); - LOG.debug("Query returned @: " + new Date()); - - aggregate(rs, startTime, endTime); - LOG.info("End aggregation cycle @ " + new Date()); - - } catch (SQLException e) { - LOG.error("Exception during aggregating metrics.", e); - success = false; - } catch (IOException e) { - LOG.error("Exception during aggregating metrics.", e); - success = false; - } finally { - if (stmt != null) { - try { - stmt.close(); - } catch (SQLException e) { - // Ignore - } - } - if (conn != null) { - try { - conn.close(); - } catch (SQLException sql) { - // Ignore - } - } - } - - LOG.info("End aggregation cycle @ " + new Date()); - return success; - } - - protected abstract PhoenixTransactSQL.Condition - prepareMetricQueryCondition(long startTime, long endTime); - - protected abstract void aggregate(ResultSet rs, long startTime, long endTime) - throws IOException, SQLException; - - protected abstract Long getSleepIntervalMillis(); - - protected abstract Integer getCheckpointCutOffMultiplier(); - - protected Long getCheckpointCutOffIntervalMillis() { - return getCheckpointCutOffMultiplier() * getSleepIntervalMillis(); - } - - protected abstract boolean isDisabled(); - - protected abstract String getCheckpointLocation(); - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/Aggregator.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/Aggregator.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/Aggregator.java deleted file mode 100644 index f514298..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/Aggregator.java +++ /dev/null @@ -1,60 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .timeline; - - -import java.util.Map; - -/** - * - */ -public class Aggregator { - - public double[] calculateAggregates(Map<Long, Double> metricValues) { - double[] values = new double[4]; - double max = Double.MIN_VALUE; - double min = Double.MAX_VALUE; - double sum = 0.0; - int metricCount = 0; - - if (metricValues != null && !metricValues.isEmpty()) { - for (Double value : metricValues.values()) { - // TODO: Some nulls in data - need to investigate null values from host - if (value != null) { - if (value > max) { - max = value; - } - if (value < min) { - min = value; - } - sum += value; - } - } - metricCount = metricValues.values().size(); - } - // BR: WHY ZERO is a good idea? - values[0] = sum; - values[1] = max != Double.MIN_VALUE ? max : 0.0; - values[2] = min != Double.MAX_VALUE ? min : 0.0; - values[3] = metricCount; - - return values; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ConnectionProvider.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ConnectionProvider.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ConnectionProvider.java deleted file mode 100644 index 47435f4..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/ConnectionProvider.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .timeline; - - -import java.sql.Connection; -import java.sql.SQLException; - -/** - * - */ -public interface ConnectionProvider { - public Connection getConnection() throws SQLException; -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/DefaultPhoenixDataSource.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/DefaultPhoenixDataSource.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/DefaultPhoenixDataSource.java deleted file mode 100644 index 652c492..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/DefaultPhoenixDataSource.java +++ /dev/null @@ -1,78 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .timeline; - - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -public class DefaultPhoenixDataSource implements ConnectionProvider { - - static final Log LOG = LogFactory.getLog(DefaultPhoenixDataSource.class); - private static final String ZOOKEEPER_CLIENT_PORT = - "hbase.zookeeper.property.clientPort"; - private static final String ZOOKEEPER_QUORUM = "hbase.zookeeper.quorum"; - private static final String ZNODE_PARENT = "zookeeper.znode.parent"; - - private static final String connectionUrl = "jdbc:phoenix:%s:%s:%s"; - private final String url; - - public DefaultPhoenixDataSource(Configuration hbaseConf) { - String zookeeperClientPort = hbaseConf.getTrimmed(ZOOKEEPER_CLIENT_PORT, - "2181"); - String zookeeperQuorum = hbaseConf.getTrimmed(ZOOKEEPER_QUORUM); - String znodeParent = hbaseConf.getTrimmed(ZNODE_PARENT, "/hbase"); - if (zookeeperQuorum == null || zookeeperQuorum.isEmpty()) { - throw new IllegalStateException("Unable to find Zookeeper quorum to " + - "access HBase store using Phoenix."); - } - - url = String.format(connectionUrl, - zookeeperQuorum, - zookeeperClientPort, - znodeParent); - } - - /** - * Get JDBC connection to HBase store. Assumption is that the hbase - * configuration is present on the classpath and loaded by the caller into - * the Configuration object. - * Phoenix already caches the HConnection between the client and HBase - * cluster. - * - * @return @java.sql.Connection - */ - public Connection getConnection() throws SQLException { - - LOG.debug("Metric store connection url: " + url); - try { - return DriverManager.getConnection(url); - } catch (SQLException e) { - LOG.warn("Unable to connect to HBase store using Phoenix.", e); - - throw e; - } - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java deleted file mode 100644 index 9364187..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/HBaseTimelineMetricStore.java +++ /dev/null @@ -1,192 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetric; -import org.apache.hadoop.metrics2.sink.timeline.TimelineMetrics; -import org.apache.hadoop.service.AbstractService; -import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse; - -import java.io.IOException; -import java.net.URL; -import java.sql.SQLException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .timeline.PhoenixTransactSQL.Condition; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .timeline.TimelineMetricConfiguration.HBASE_SITE_CONFIGURATION_FILE; -import static org.apache.hadoop.yarn.server.applicationhistoryservice.metrics - .timeline.TimelineMetricConfiguration.METRICS_SITE_CONFIGURATION_FILE; - -public class HBaseTimelineMetricStore extends AbstractService - implements TimelineMetricStore { - - static final Log LOG = LogFactory.getLog(HBaseTimelineMetricStore.class); - private PhoenixHBaseAccessor hBaseAccessor; - - /** - * Construct the service. - * - */ - public HBaseTimelineMetricStore() { - super(HBaseTimelineMetricStore.class.getName()); - } - - @Override - protected void serviceInit(Configuration conf) throws Exception { - ClassLoader classLoader = Thread.currentThread().getContextClassLoader(); - if (classLoader == null) { - classLoader = getClass().getClassLoader(); - } - URL hbaseResUrl = classLoader.getResource(HBASE_SITE_CONFIGURATION_FILE); - URL amsResUrl = classLoader.getResource(METRICS_SITE_CONFIGURATION_FILE); - LOG.info("Found hbase site configuration: " + hbaseResUrl); - LOG.info("Found metric service configuration: " + amsResUrl); - - if (hbaseResUrl == null) { - throw new IllegalStateException("Unable to initialize the metrics " + - "subsystem. No hbase-site present in the classpath."); - } - - if (amsResUrl == null) { - throw new IllegalStateException("Unable to initialize the metrics " + - "subsystem. No ams-site present in the classpath."); - } - - Configuration hbaseConf = new Configuration(true); - hbaseConf.addResource(hbaseResUrl.toURI().toURL()); - Configuration metricsConf = new Configuration(true); - metricsConf.addResource(amsResUrl.toURI().toURL()); - - initializeSubsystem(hbaseConf, metricsConf); - } - - private void initializeSubsystem(Configuration hbaseConf, - Configuration metricsConf) { - hBaseAccessor = new PhoenixHBaseAccessor(hbaseConf, metricsConf); - hBaseAccessor.initMetricSchema(); - - // Start the cluster aggregator - TimelineMetricClusterAggregator minuteClusterAggregator = - new TimelineMetricClusterAggregator(hBaseAccessor, metricsConf); - if (!minuteClusterAggregator.isDisabled()) { - Thread aggregatorThread = new Thread(minuteClusterAggregator); - aggregatorThread.start(); - } - - // Start the cluster aggregator hourly - TimelineMetricClusterAggregatorHourly hourlyClusterAggregator = - new TimelineMetricClusterAggregatorHourly(hBaseAccessor, metricsConf); - if (!hourlyClusterAggregator.isDisabled()) { - Thread aggregatorThread = new Thread(hourlyClusterAggregator); - aggregatorThread.start(); - } - - // Start the 5 minute aggregator - TimelineMetricAggregator minuteHostAggregator = - TimelineMetricAggregatorFactory.createTimelineMetricAggregatorMinute - (hBaseAccessor, metricsConf); - if (!minuteHostAggregator.isDisabled()) { - Thread minuteAggregatorThread = new Thread(minuteHostAggregator); - minuteAggregatorThread.start(); - } - - // Start hourly host aggregator - TimelineMetricAggregator hourlyHostAggregator = - TimelineMetricAggregatorFactory.createTimelineMetricAggregatorHourly - (hBaseAccessor, metricsConf); - if (!hourlyHostAggregator.isDisabled()) { - Thread aggregatorHourlyThread = new Thread(hourlyHostAggregator); - aggregatorHourlyThread.start(); - } - } - - @Override - protected void serviceStop() throws Exception { - super.serviceStop(); - } - - //TODO: update to work with HOSTS_COUNT and METRIC_COUNT - @Override - public TimelineMetrics getTimelineMetrics(List<String> metricNames, - String hostname, String applicationId, String instanceId, - Long startTime, Long endTime, Integer limit, - boolean groupedByHosts) throws SQLException, IOException { - - Condition condition = new Condition(metricNames, hostname, applicationId, - instanceId, startTime, endTime, limit, groupedByHosts); - - if (hostname == null) { - return hBaseAccessor.getAggregateMetricRecords(condition); - } - - return hBaseAccessor.getMetricRecords(condition); - } - - @Override - public TimelineMetric getTimelineMetric(String metricName, String hostname, - String applicationId, String instanceId, Long startTime, - Long endTime, Integer limit) - throws SQLException, IOException { - - TimelineMetrics metrics = hBaseAccessor.getMetricRecords( - new Condition(Collections.singletonList(metricName), hostname, - applicationId, instanceId, startTime, endTime, limit, true) - ); - - TimelineMetric metric = new TimelineMetric(); - List<TimelineMetric> metricList = metrics.getMetrics(); - - if (metricList != null && !metricList.isEmpty()) { - metric.setMetricName(metricList.get(0).getMetricName()); - metric.setAppId(metricList.get(0).getAppId()); - metric.setInstanceId(metricList.get(0).getInstanceId()); - metric.setHostName(metricList.get(0).getHostName()); - // Assumption that metrics are ordered by start time - metric.setStartTime(metricList.get(0).getStartTime()); - Map<Long, Double> metricRecords = new HashMap<Long, Double>(); - for (TimelineMetric timelineMetric : metricList) { - metricRecords.putAll(timelineMetric.getMetricValues()); - } - metric.setMetricValues(metricRecords); - } - - return metric; - } - - - @Override - public TimelinePutResponse putMetrics(TimelineMetrics metrics) - throws SQLException, IOException { - - // Error indicated by the Sql exception - TimelinePutResponse response = new TimelinePutResponse(); - - hBaseAccessor.insertMetricRecords(metrics); - - return response; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricAggregate.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricAggregate.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricAggregate.java deleted file mode 100644 index 61e15d7..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricAggregate.java +++ /dev/null @@ -1,110 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.codehaus.jackson.annotate.JsonProperty; -import org.codehaus.jackson.annotate.JsonSubTypes; -import org.codehaus.jackson.map.ObjectMapper; - -import java.io.IOException; - -/** -* -*/ -@JsonSubTypes({@JsonSubTypes.Type(value = MetricClusterAggregate.class), - @JsonSubTypes.Type(value = MetricHostAggregate.class)}) -@InterfaceAudience.Public -@InterfaceStability.Unstable -public class MetricAggregate { - private static final ObjectMapper mapper = new ObjectMapper(); - - protected Double sum = 0.0; - protected Double deviation; - protected Double max = Double.MIN_VALUE; - protected Double min = Double.MAX_VALUE; - - public MetricAggregate() { - } - - MetricAggregate(Double sum, Double deviation, Double max, - Double min) { - this.sum = sum; - this.deviation = deviation; - this.max = max; - this.min = min; - } - - void updateSum(Double sum) { - this.sum += sum; - } - - void updateMax(Double max) { - if (max > this.max) { - this.max = max; - } - } - - void updateMin(Double min) { - if (min < this.min) { - this.min = min; - } - } - - @JsonProperty("sum") - Double getSum() { - return sum; - } - - @JsonProperty("deviation") - Double getDeviation() { - return deviation; - } - - @JsonProperty("max") - Double getMax() { - return max; - } - - @JsonProperty("min") - Double getMin() { - return min; - } - - public void setSum(Double sum) { - this.sum = sum; - } - - public void setDeviation(Double deviation) { - this.deviation = deviation; - } - - public void setMax(Double max) { - this.max = max; - } - - public void setMin(Double min) { - this.min = min; - } - - public String toJSON() throws IOException { - return mapper.writeValueAsString(this); - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/ba3d6926/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricClusterAggregate.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricClusterAggregate.java b/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricClusterAggregate.java deleted file mode 100644 index c13c85f..0000000 --- a/ambari-metrics/ambari-metrics-hadoop-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/timeline/MetricClusterAggregate.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.timeline; - - -import org.codehaus.jackson.annotate.JsonCreator; -import org.codehaus.jackson.annotate.JsonProperty; - -/** -* -*/ -public class MetricClusterAggregate extends MetricAggregate { - private int numberOfHosts; - - @JsonCreator - public MetricClusterAggregate() { - } - - MetricClusterAggregate(Double sum, int numberOfHosts, Double deviation, - Double max, Double min) { - super(sum, deviation, max, min); - this.numberOfHosts = numberOfHosts; - } - - @JsonProperty("numberOfHosts") - int getNumberOfHosts() { - return numberOfHosts; - } - - void updateNumberOfHosts(int count) { - this.numberOfHosts += count; - } - - public void setNumberOfHosts(int numberOfHosts) { - this.numberOfHosts = numberOfHosts; - } - - /** - * Find and update min, max and avg for a minute - */ - void updateAggregates(MetricClusterAggregate hostAggregate) { - updateMax(hostAggregate.getMax()); - updateMin(hostAggregate.getMin()); - updateSum(hostAggregate.getSum()); - updateNumberOfHosts(hostAggregate.getNumberOfHosts()); - } - - @Override - public String toString() { -// MetricClusterAggregate - return "MetricAggregate{" + - "sum=" + sum + - ", numberOfHosts=" + numberOfHosts + - ", deviation=" + deviation + - ", max=" + max + - ", min=" + min + - '}'; - } -}
