AMBARI-13926. AMS log file not getting updated or created on fresh deploy. (Aravindan Vijayan via swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/05a4b66b Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/05a4b66b Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/05a4b66b Branch: refs/heads/branch-2.1 Commit: 05a4b66bd1d3ca0ba7b2a42e7df7f0872986d82f Parents: 9cf3bca Author: Siddharth Wagle <swa...@hortonworks.com> Authored: Tue Nov 17 14:58:26 2015 -0800 Committer: Siddharth Wagle <swa...@hortonworks.com> Committed: Tue Nov 17 14:58:26 2015 -0800 ---------------------------------------------------------------------- .../ambari-metrics-timelineservice/pom.xml | 2 + .../jmetertest/AMSJMeterLoadTest.java | 202 --- .../loadsimulator/jmetertest/AppGetMetric.java | 57 - .../jmetertest/GetMetricRequestInfo.java | 61 - .../jmetertest/JmeterTestPlanTask.java | 269 ---- .../src/main/resources/loadsimulator/README | 65 - .../loadsimulator/ams-jmeter.properties | 56 - .../resources/loadsimulator/amsJmeterGraph.jmx | 104 -- .../resources/loadsimulator/jmeter.properties | 1172 ------------------ .../loadsimulator/saveservice.properties | 381 ------ .../main/resources/ui_metrics_def/AMS-HBASE.dat | 26 - .../main/resources/ui_metrics_def/DATANODE.dat | 4 - .../resources/ui_metrics_def/FLUME_HANDLER.dat | 63 - .../src/main/resources/ui_metrics_def/HBASE.dat | 47 - .../src/main/resources/ui_metrics_def/HOST.dat | 79 -- .../resources/ui_metrics_def/KAFKA_BROKER.dat | 16 - .../main/resources/ui_metrics_def/NAMENODE.dat | 30 - .../main/resources/ui_metrics_def/NIMBUS.dat | 28 - .../resources/ui_metrics_def/NODEMANAGER.dat | 33 - .../ui_metrics_def/RESOURCEMANAGER.dat | 11 - .../jmetertest/AMSJMeterLoadTest.java | 202 +++ .../jmetertest/jmetertest/AppGetMetric.java | 57 + .../jmetertest/GetMetricRequestInfo.java | 61 + .../jmetertest/JmeterTestPlanTask.java | 276 +++++ .../src/test/resources/loadsimulator/README | 65 + .../loadsimulator/ams-jmeter.properties | 56 + .../resources/loadsimulator/amsJmeterGraph.jmx | 104 ++ .../resources/loadsimulator/jmeter.properties | 1172 ++++++++++++++++++ .../loadsimulator/saveservice.properties | 381 ++++++ .../test/resources/ui_metrics_def/AMS-HBASE.dat | 26 + .../test/resources/ui_metrics_def/DATANODE.dat | 4 + .../resources/ui_metrics_def/FLUME_HANDLER.dat | 63 + .../src/test/resources/ui_metrics_def/HBASE.dat | 47 + .../src/test/resources/ui_metrics_def/HOST.dat | 79 ++ .../resources/ui_metrics_def/KAFKA_BROKER.dat | 16 + .../test/resources/ui_metrics_def/NAMENODE.dat | 30 + .../test/resources/ui_metrics_def/NIMBUS.dat | 28 + .../resources/ui_metrics_def/NODEMANAGER.dat | 33 + .../ui_metrics_def/RESOURCEMANAGER.dat | 11 + 39 files changed, 2713 insertions(+), 2704 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml index 1f76ba2..7b4afb9 100644 --- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml +++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml @@ -413,6 +413,7 @@ <groupId>org.apache.jmeter</groupId> <artifactId>ApacheJMeter_core</artifactId> <version>2.13</version> + <scope>test</scope> <exclusions> <exclusion> <groupId>org.slf4j</groupId> @@ -432,6 +433,7 @@ <groupId>org.apache.jmeter</groupId> <artifactId>ApacheJMeter_http</artifactId> <version>2.13</version> + <scope>test</scope> <exclusions> <exclusion> <groupId>org.slf4j</groupId> http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AMSJMeterLoadTest.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AMSJMeterLoadTest.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AMSJMeterLoadTest.java deleted file mode 100644 index 187c3f1..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AMSJMeterLoadTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest; - -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.MetricsLoadSimulator; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -public class AMSJMeterLoadTest { - - private final static Logger LOG = LoggerFactory.getLogger(AMSJMeterLoadTest.class); - private static String PROPERTIES_FILE = "loadsimulator/ams-jmeter.properties"; - private ScheduledExecutorService scheduledExecutorService = null; - private List<AppGetMetric> appGetMetrics; - private Properties amsJmeterProperties = null; - - public AMSJMeterLoadTest(String testType, String userDefinedPropertiesFile) { - - if (null == userDefinedPropertiesFile || userDefinedPropertiesFile.isEmpty()) { - this.amsJmeterProperties = readProperties(PROPERTIES_FILE); - } else { - this.amsJmeterProperties = readProperties(userDefinedPropertiesFile); - } - - if ("U".equals(testType)) { //GET metrics simulator - int numInstances = Integer.valueOf(amsJmeterProperties.getProperty("num-ui-instances")); - this.scheduledExecutorService = Executors.newScheduledThreadPool(numInstances); - this.appGetMetrics = initializeGetMetricsPayload(amsJmeterProperties); - this.runTest(numInstances); - } else { //PUT Metrics simulator - Map<String, String> mapArgs = new HashMap<String, String>(); - mapArgs.put("hostName", amsJmeterProperties.getProperty("host-prefix")); - mapArgs.put("numberOfHosts", amsJmeterProperties.getProperty("num-hosts")); - mapArgs.put("metricsHostName", amsJmeterProperties.getProperty("ams-host-port")); - mapArgs.put("collectInterval", amsJmeterProperties.getProperty("collection-interval")); - mapArgs.put("sendInterval", amsJmeterProperties.getProperty("send-interval")); - mapArgs.put("master", amsJmeterProperties.getProperty("create-master")); - MetricsLoadSimulator.startTest(mapArgs); - } - } - - public static Properties readProperties(String propertiesFile) { - try { - Properties properties = new Properties(); - InputStream inputStream = ClassLoader.getSystemResourceAsStream(propertiesFile); - if (inputStream == null) { - inputStream = new FileInputStream(propertiesFile); - } - properties.load(inputStream); - return properties; - } catch (IOException ioEx) { - LOG.error("Error reading properties file for jmeter"); - return null; - } - } - - private static List<GetMetricRequestInfo> readMetricsFromFile(String app) { - InputStream input = null; - List<GetMetricRequestInfo> metricList = new ArrayList<>(); - String fileName = "ui_metrics_def/" + app + ".dat"; - - try { - input = ClassLoader.getSystemResourceAsStream(fileName); - BufferedReader reader = new BufferedReader(new InputStreamReader(input)); - String line; - List<String> metrics = new ArrayList<>(); - while ((line = reader.readLine()) != null) { - - if (line.startsWith("|")) { - boolean needsTimestamps = line.contains("startTime"); - boolean needsHost = line.contains("hostname"); - metricList.add(new GetMetricRequestInfo(metrics, needsTimestamps, needsHost)); - metrics.clear(); - } else { - metrics.add(line); - } - } - return metricList; - } catch (IOException e) { - LOG.error("Cannot read file " + fileName + " for appID " + app, e); - } finally { - if (input != null) { - try { - input.close(); - } catch (IOException ex) { - } - } - } - return null; - } - - private static List<AppGetMetric> initializeGetMetricsPayload(Properties amsJmeterProperties) { - - List<AppGetMetric> appGetMetrics = new ArrayList<AppGetMetric>(); - String appsToTest = amsJmeterProperties.getProperty("apps-to-test"); - String[] apps; - - if (appsToTest != null && !appsToTest.isEmpty()) { - apps = StringUtils.split(appsToTest, ","); - } else { - apps = new String[JmeterTestPlanTask.ClientApp.values().length]; - int ctr = 0; - for (JmeterTestPlanTask.ClientApp app : JmeterTestPlanTask.ClientApp.values()) - apps[ctr++] = app.getId(); - } - - for (String app : apps) { - - int interval = Integer.valueOf(amsJmeterProperties.getProperty("get-interval")); - String intervalString = amsJmeterProperties.getProperty(app + "-get-interval"); - if (intervalString != null && !intervalString.isEmpty()) { - interval = Integer.valueOf(intervalString); - } - appGetMetrics.add(new AppGetMetric(readMetricsFromFile(app), interval, app)); - } - - return appGetMetrics; - } - - public void runTest(int numInstances) { - - int appRefreshRate = Integer.valueOf(amsJmeterProperties.getProperty("app-refresh-rate")); - for (int i = 0; i < numInstances; i++) { - ScheduledFuture future = scheduledExecutorService.scheduleAtFixedRate(new JmeterTestPlanTask(appGetMetrics, - amsJmeterProperties), 0, appRefreshRate, TimeUnit.MILLISECONDS); - } - } - - /** - * Sample Usage: - * java -cp "lib/*":ambari-metrics-timelineservice-2.1.1.0.jar org.apache.hadoop.yarn.server.applicationhistoryservice - * .metrics.loadsimulator.jmeter.AMSJMeterLoadTest - * -t UI -p ambari-metrics-timelineservice/src/main/resources/jmeter/ams-jmeter.properties - */ - public static void main(String[] args) { - Map<String, String> mapArgs = parseArgs(args); - String testType = mapArgs.get("type"); - String amsJmeterPropertiesFile = mapArgs.get("amsJmeterPropertiesFile"); - new AMSJMeterLoadTest(testType, amsJmeterPropertiesFile); - } - - private static Map<String, String> parseArgs(String[] args) { - Map<String, String> mapProps = new HashMap<String, String>(); - if (args.length == 0) { - printUsage(); - throw new RuntimeException("Unexpected argument, See usage message."); - } else { - for (int i = 0; i < args.length; i += 2) { - String arg = args[i]; - if (arg.equals("-t")) { - mapProps.put("type", args[i + 1]); - } else if (arg.equals("-p")) { - mapProps.put("amsJmeterPropertiesFile", args[i + 1]); - } else { - printUsage(); - throw new IllegalArgumentException("Unexpected argument, See usage message."); - } - } - } - return mapProps; - } - - public static void printUsage() { - System.err.println("Usage: java AMSJmeterLoadTest [OPTIONS]"); - System.err.println("Options: "); - System.err.println("[-t type (S=>Sink/U=>UI)] [-p amsJmeterPropertiesFile (Optional)]"); - } - -} - - http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AppGetMetric.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AppGetMetric.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AppGetMetric.java deleted file mode 100644 index 727a1c7..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/AppGetMetric.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest; - -import java.util.List; - -public class AppGetMetric { - - private String app; - private int interval; - private List<GetMetricRequestInfo> requests; - - public AppGetMetric(List<GetMetricRequestInfo> requests, int interval, String app) { - this.setMetricRequests(requests); - this.setInterval(interval); - this.setApp(app); - } - - public List<GetMetricRequestInfo> getMetricRequests() { - return requests; - } - - public void setMetricRequests(List<GetMetricRequestInfo> requests) { - this.requests = requests; - } - - public int getInterval() { - return interval; - } - - public void setInterval(int interval) { - this.interval = interval; - } - - public String getApp() { - return app; - } - - public void setApp(String app) { - this.app = app; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/GetMetricRequestInfo.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/GetMetricRequestInfo.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/GetMetricRequestInfo.java deleted file mode 100644 index 26c5025..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/GetMetricRequestInfo.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest; - -import org.apache.commons.lang.StringUtils; - -import java.util.List; - - -public class GetMetricRequestInfo { - - private String metricStringPayload; - private boolean needsTimestamps; - private boolean needsHost; - - public GetMetricRequestInfo(List<String> metrics, boolean needsTimestamps, boolean needsHost) { - - this.setMetricStringPayload(StringUtils.join(metrics, ",")); - this.setNeedsTimestamps(needsTimestamps); - this.setNeedsHost(needsHost); - } - - public String getMetricStringPayload() { - return metricStringPayload; - } - - public void setMetricStringPayload(String metricStringPayload) { - this.metricStringPayload = metricStringPayload; - } - - public boolean needsTimestamps() { - return needsTimestamps; - } - - public void setNeedsTimestamps(boolean needsTimestamps) { - this.needsTimestamps = needsTimestamps; - } - - public boolean needsHost() { - return needsHost; - } - - public void setNeedsHost(boolean needsHost) { - this.needsHost = needsHost; - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/JmeterTestPlanTask.java ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/JmeterTestPlanTask.java b/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/JmeterTestPlanTask.java deleted file mode 100644 index f7e27b1..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/metrics/loadsimulator/jmetertest/JmeterTestPlanTask.java +++ /dev/null @@ -1,269 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmetertest; - -import org.apache.commons.io.IOUtils; -import org.apache.jmeter.control.LoopController; -import org.apache.jmeter.engine.StandardJMeterEngine; -import org.apache.jmeter.protocol.http.sampler.HTTPSampler; -import org.apache.jmeter.protocol.http.util.HTTPConstants; -import org.apache.jmeter.reporters.ResultCollector; -import org.apache.jmeter.reporters.Summariser; -import org.apache.jmeter.testelement.TestElement; -import org.apache.jmeter.testelement.TestPlan; -import org.apache.jmeter.threads.JMeterContextService; -import org.apache.jmeter.threads.ThreadGroup; -import org.apache.jmeter.timers.ConstantTimer; -import org.apache.jmeter.util.JMeterUtils; -import org.apache.jorphan.collections.HashTree; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.lang.reflect.Field; -import java.util.*; - -public class JmeterTestPlanTask implements Runnable { - - private static StandardJMeterEngine jmeterEngine = null; - private final static Logger LOG = LoggerFactory.getLogger(JmeterTestPlanTask.class); - private List<AppGetMetric> appGetMetrics; - private Properties amsJmeterProperties; - private HashTree amsTestPlanTree; - private TestPlan amsTestPlan; - private static final String JMETER_HOME = "loadsimulator"; - private static final String JMETER_PROPERTIES_FILE = JMETER_HOME + "/jmeter.properties"; - private static final String SAVESERVICE_PROPERTIES_FILE = JMETER_HOME + "/saveservice.properties"; - - public enum ClientApp { - HOST("HOST"), - NAMENODE("NAMENODE"), - HBASE("HBASE"), - NIMBUS("NIMBUS"), - KAFKA_BROKER("KAFKA_BROKER"), - FLUME_HANDLER("FLUME_HANDLER"), - AMS_HBASE("AMS-HBASE"), - NODEMANAGER("NODEMANAGER"), - RESOURCEMANAGER("RESOURCEMANAGER"), - DATANODE("DATANODE"); - - private String id; - - private ClientApp(String id) { - this.id = id; - } - - public String getId() { - return id; - } - } - - public JmeterTestPlanTask(List<AppGetMetric> appGetMetrics, Properties amsJmeterProperties) { - this.appGetMetrics = appGetMetrics; - this.amsJmeterProperties = amsJmeterProperties; - amsTestPlanTree = new HashTree(); - amsTestPlan = new TestPlan("AMS JMeter Load Test plan"); - System.out.println("Starting AMS Jmeter load testing"); - } - - public void run() { - if (jmeterEngine != null) { - - Object[] threadGroups = amsTestPlanTree.getArray(amsTestPlan); - for (Object threadGroupObj : threadGroups) { - if (threadGroupObj instanceof ThreadGroup) { - ThreadGroup threadGroup = (ThreadGroup) threadGroupObj; - threadGroup.stop(); - } - } - amsTestPlanTree.clear(); - jmeterEngine.askThreadsToStop(); - jmeterEngine.stopTest(); - JMeterContextService.endTest(); - } - - //Start the new test plan for the new app. - try { - //Initialize Jmeter essentials - jmeterEngine = new StandardJMeterEngine(); - JMeterContextService.getContext().setEngine(jmeterEngine); - - //Workaround to supply JMeterUtils with jmeter.prooperties from JAR. - JMeterUtils.setJMeterHome(""); - Field f = new JMeterUtils().getClass().getDeclaredField("appProperties"); - f.setAccessible(true); - f.set(null, AMSJMeterLoadTest.readProperties(JMETER_PROPERTIES_FILE)); - - //Copy saveservices.properties file to tmp dir for JMeter to consume. - InputStream inputStream = ClassLoader.getSystemResourceAsStream(SAVESERVICE_PROPERTIES_FILE); - if (inputStream == null) { - inputStream = new FileInputStream(SAVESERVICE_PROPERTIES_FILE); - } - String tmpDir = System.getProperty("java.io.tmpdir"); - OutputStream outputStream = new FileOutputStream(tmpDir + "/saveservice.properties"); - IOUtils.copy(inputStream, outputStream); - outputStream.close(); - JMeterUtils.setProperty("saveservice_properties", tmpDir + "/saveservice.properties"); - - //Initialize Test plan - amsTestPlan.setProperty(TestElement.TEST_CLASS, TestPlan.class.getName()); - amsTestPlanTree.add("AMS Test plan", amsTestPlan); - - //Choose a random APP to run the perform GET metrics request. - int currentAppIndex = new Random().nextInt(appGetMetrics.size()); - - //Create ThreadGroup for the App - createThreadGroupHashTree(currentAppIndex, amsJmeterProperties, amsTestPlanTree, amsTestPlan); - - //Geneates the JMX file that you can use through the GUI mode. - //SaveService.saveTree(amsTestPlanTree, new FileOutputStream(JMETER_HOME + "/" + "amsTestPlan.jmx")); - - //Summarizer output to get test progress in stdout like. - Summariser summariser = null; - String summariserName = JMeterUtils.getPropDefault("summariser.name", "summary"); - if (summariserName.length() > 0) { - summariser = new Summariser(summariserName); - } - - //Store execution results into a .jtl file - String jmeterLogFile = tmpDir + "/amsJmeterTestResults.jtl"; - ResultCollector resultCollector = new ResultCollector(summariser); - resultCollector.setFilename(jmeterLogFile); - amsTestPlanTree.add(amsTestPlanTree.getArray()[0], resultCollector); - jmeterEngine.configure(amsTestPlanTree); - jmeterEngine.run(); - - LOG.info("AMS Jmeter Test started up successfully"); - - } catch (Exception ioEx) { - amsTestPlanTree.clear(); - jmeterEngine.askThreadsToStop(); - jmeterEngine.stopTest(); - JMeterContextService.endTest(); - LOG.error("Error occurred while running AMS load test : " + ioEx.getMessage()); - ioEx.printStackTrace(); - } - } - - private ConstantTimer createConstantTimer(int delay) { - ConstantTimer timer = new ConstantTimer(); - timer.setDelay("" + delay); - return timer; - } - - private Map<String, String> getAppSpecificParameters(String app, GetMetricRequestInfo request, Properties amsJmeterProperties) { - - Map<String, String> parametersMap = new HashMap<String, String>(); - String hostPrefix = amsJmeterProperties.getProperty("host-prefix"); - String hostSuffix = amsJmeterProperties.getProperty("host-suffix"); - int minHostIndex = Integer.valueOf(amsJmeterProperties.getProperty("min-host-index")); - int numHosts = Integer.valueOf(amsJmeterProperties.getProperty("num-hosts")); - - parametersMap.put("appId", app); - - if (request.needsTimestamps()) { - long currentTime = System.currentTimeMillis(); - long oneHourBack = currentTime - 3600 * 1000; - parametersMap.put("startTime", String.valueOf(oneHourBack)); - parametersMap.put("endTime", String.valueOf(currentTime)); - } - - if (request.needsHost()) { - if (ClientApp.AMS_HBASE.getId().equals(app)) { - parametersMap.put("hostname", amsJmeterProperties.getProperty("ams-host")); - } else if (ClientApp.HOST.getId().equals(app) || ClientApp.NODEMANAGER.getId().equals(app)) { - int randomHost = minHostIndex + new Random().nextInt(numHosts); - parametersMap.put("hostname", hostPrefix + randomHost + hostSuffix); - } else { - parametersMap.put("hostname", hostPrefix + amsJmeterProperties.getProperty(app + "-host") + hostSuffix); - } - } - parametersMap.put("metricNames", request.getMetricStringPayload()); - return parametersMap; - } - - private void createThreadGroupHashTree(int appIndex, Properties amsJmeterProperties, HashTree amsTestPlanTree, TestPlan amsTestPlan) { - - AppGetMetric appGetMetric = appGetMetrics.get(appIndex); - String app = appGetMetric.getApp(); - int interval = appGetMetric.getInterval(); - - //Read and validate AMS information. - String[] amsHostPort = amsJmeterProperties.getProperty("ams-host-port").split(":"); - String amsHost = amsHostPort[0]; - String amsPath = amsJmeterProperties.getProperty("ams-path"); - int amsPort = Integer.valueOf(amsHostPort[1]); - int numLoops = Integer.valueOf(amsJmeterProperties.getProperty("num-get-calls-per-app")); - - LoopController loopController = createLoopController(app + " GET loop controller", numLoops, false); - for (GetMetricRequestInfo request : appGetMetric.getMetricRequests()) { - - ThreadGroup threadGroup = createThreadGroup(app + " GET threadGroup", 1, 0, loopController); - - HashTree threadGroupHashTree = amsTestPlanTree.add(amsTestPlan, threadGroup); - Map<String, String> parametersMap = getAppSpecificParameters(app, request, amsJmeterProperties); - - HTTPSampler sampler = createGetSampler("GET " + app + " metrics", amsHost, amsPort, amsPath, null, parametersMap); - - if (numLoops > 1) { - threadGroupHashTree.add(createConstantTimer(interval)); - } - - threadGroupHashTree.add(sampler); - } - } - - private HTTPSampler createGetSampler(String name, String domain, int port, String path, String encoding, Map<String, String> parameters) { - - HTTPSampler sampler = new HTTPSampler(); - sampler.setDomain(domain); - sampler.setPort(port); - sampler.setPath(path); - sampler.setMethod(HTTPConstants.GET); - - if (encoding != null) - sampler.setContentEncoding(encoding); - - for (Map.Entry<String, String> entry : parameters.entrySet()) { - sampler.addArgument(entry.getKey(), entry.getValue()); - } - sampler.setName(name); - return sampler; - } - - private LoopController createLoopController(String name, int numLoops, boolean continueForever) { - LoopController loopController = new LoopController(); - loopController.setLoops(numLoops); - loopController.setProperty(TestElement.TEST_CLASS, LoopController.class.getName()); - loopController.initialize(); - loopController.setContinueForever(continueForever); - loopController.setName(name); - return loopController; - } - - private ThreadGroup createThreadGroup(String name, int numThreads, int rampUp, LoopController loopController) { - ThreadGroup threadGroup = new ThreadGroup(); - threadGroup.setName(name); - threadGroup.setNumThreads(numThreads); - threadGroup.setRampUp(rampUp); - threadGroup.setSamplerController(loopController); - threadGroup.setProperty(TestElement.TEST_CLASS, ThreadGroup.class.getName()); - return threadGroup; - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/README ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/README b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/README deleted file mode 100644 index 39e5365..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/README +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - WORKING OF THE AMS LOAD SIMULATOR - The AMS load simulator is designed to perform load testing on a live AMS instance by performing the role of either a - sink or an Ambari UI instance based on how it is being invoked. - - > When it acts as a host sink, it makes calls to the AMS to PUT metrics for all the services for a defined interval. - The simulator can also be used to start up "N" such host instances where each instance has a number of Sinks that PUT - metrics. - > When the load simulator is invoked as a UI instance, it makes GET metrics calls to the AMS in defined - intervals for all the services. The rate of the GET metrics call and the list of metrics requested has been designed - to closely match an actual Ambari UI instance. Apache JMeter API has been used to design the GET calls made to the - AMS. - - The load simulator uses a properties file (ams-jmeter.properties) file to configure the test run. It is part of the - JAR in the same folder as this README file. It can also be supplied as a command line argument to the test using the - "-p" option. Other properties files like jmeter.properties and saveservice.properties contain JMeter internal - properties and need not be modified. - - - INSTRUCTIONS TO RUN THE SIMULATOR - 1. Modify the ams-jmeter.properties to point to your AMS host. Change the properties "num-hosts" based on how many hosts - need to be simulated for sinks. The GET Metric section of the properties is used for fine tuning GET call interval - for every APP type. - 2. Build the ambari-metrics-timelineservice jar. - 3. Invoke the test using the command as follows. - - java -cp lib/*:ambari-metrics-timelineservice-<version>.jar org.apache.hadoop.yarn.server.applicationhistoryservice.metrics.loadsimulator.jmeter.AMSJMeterLoadTest -t <S/U> - - lib/* -> All the dependency JARs generated for the ambari-metrics-timelineservice JAR. - -t option => S-Sink simulator or U-UI simulator - You can use the -p <location of ams-jmeter.properties> option to pass in your own properties file. - - 4. Test results will be found at <TMP_DIR>/amsTestResults.jtl. - 5. Open the amsJmeterGrpah.jmx file through a JMeter GUI instance and supply the results (amsTestResults.jtl) file as - input to the Graph to be drawn. - - TESTING ON GCE - - 1. Copy the JAR, libs, optional ams-jmeter.properties file to all the machines on which the test needs to be run. - 2. Sink simulation for num-hosts = N. - Start the test with -t S on 1 machine with property "create-master=true". - Start the test with -t S on N-1 machines with property "create-master=false" - 3. UI simulation - Start the test with -t U on 1 or more machines. - 4. To stop the test after you have sufficient load testing done, the following command should be run on all machines. - ps axf | grep jmeter | grep -v grep | awk '{print "kill -9 " $1}' | sh - 5. Copy over the results file to a location with JMeter downloaded. Open the amsJmeterGraph.jmx on jmeter and browse to - open the results file as input to the graph. http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/ams-jmeter.properties ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/ams-jmeter.properties b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/ams-jmeter.properties deleted file mode 100644 index 3353d43..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/ams-jmeter.properties +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#AMS information -ams-host-port=104.196.94.27:6188 -ams-path=/ws/v1/timeline/metrics - -# Host Information, using the host-prefix, num-hosts and <MASTER>-host -# Hosts in this case will be TestHost0, TestHost1 -# For example NAMENODE, NIMBUS,HBASE MASTER,RESOURCEMANAGER host will be TestHost0 -host-prefix=TestHost -host-suffix= -min-host-index=0 -num-hosts=2 -NAMENODE-host=0 -NIMBUS-host=0 -HBASE-host=0 -RESOURCEMANAGER-host=0 - -# PUT Metric / Sinks config -collection-interval=1000 -send-interval=3000 -create-master=true - -# GET Metric / Client Apps config -num-ui-instances=1 -apps-to-test= -app-refresh-rate=8000 -num-get-calls-per-app=3 -get-interval=3000 -HOST-get-interval=3000 -NAMENODE-get-interval=2000 -HBASE-get-interval=3000 -NIMBUS-get-interval=2000 -AMS-HBASE-get-interval=2000 -FLUME_HANDLER-get-interval=2000 -NODEMANAGER-get-interval=2000 -KAFKA_BROKER-get-interval=2000 -DATANODE-get-interval=3000 -RESOURCEMANAGER-get-interval=3000 - http://git-wip-us.apache.org/repos/asf/ambari/blob/05a4b66b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/amsJmeterGraph.jmx ---------------------------------------------------------------------- diff --git a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/amsJmeterGraph.jmx b/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/amsJmeterGraph.jmx deleted file mode 100644 index 06d6360..0000000 --- a/ambari-metrics/ambari-metrics-timelineservice/src/main/resources/loadsimulator/amsJmeterGraph.jmx +++ /dev/null @@ -1,104 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one - ~ or more contributor license agreements. See the NOTICE file - ~ distributed with this work for additional information - ~ regarding copyright ownership. The ASF licenses this file - ~ to you under the Apache License, Version 2.0 (the - ~ "License"); you may not use this file except in compliance - ~ with the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> -<jmeterTestPlan version="1.2" properties="2.8" jmeter="2.13 r1665067"> - <hashTree> - <TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="Test Plan" enabled="true"> - <stringProp name="TestPlan.comments"></stringProp> - <boolProp name="TestPlan.functional_mode">false</boolProp> - <boolProp name="TestPlan.serialize_threadgroups">false</boolProp> - <elementProp name="TestPlan.user_defined_variables" elementType="Arguments" guiclass="ArgumentsPanel" testclass="Arguments" testname="User Defined Variables" enabled="true"> - <collectionProp name="Arguments.arguments"/> - </elementProp> - <stringProp name="TestPlan.user_define_classpath"></stringProp> - </TestPlan> - <hashTree> - <ResultCollector guiclass="GraphVisualizer" testclass="ResultCollector" testname="Latency Graph" enabled="true"> - <boolProp name="ResultCollector.error_logging">false</boolProp> - <objProp> - <name>saveConfig</name> - <value class="SampleSaveConfiguration"> - <time>false</time> - <latency>false</latency> - <timestamp>true</timestamp> - <success>true</success> - <label>true</label> - <code>true</code> - <message>true</message> - <threadName>false</threadName> - <dataType>true</dataType> - <encoding>false</encoding> - <assertions>false</assertions> - <subresults>false</subresults> - <responseData>true</responseData> - <samplerData>true</samplerData> - <xml>true</xml> - <fieldNames>true</fieldNames> - <responseHeaders>true</responseHeaders> - <requestHeaders>true</requestHeaders> - <responseDataOnError>true</responseDataOnError> - <saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage> - <assertionsResultsToSave>0</assertionsResultsToSave> - <url>true</url> - <hostname>true</hostname> - <sampleCount>true</sampleCount> - </value> - </objProp> - <stringProp name="filename">/tmp/amsJmeterTestResults.jtl</stringProp> - </ResultCollector> - <hashTree/> - <ResultCollector guiclass="RespTimeGraphVisualizer" testclass="ResultCollector" testname="Response Time Graph" enabled="true"> - <boolProp name="ResultCollector.error_logging">false</boolProp> - <objProp> - <name>saveConfig</name> - <value class="SampleSaveConfiguration"> - <time>false</time> - <latency>false</latency> - <timestamp>true</timestamp> - <success>true</success> - <label>true</label> - <code>true</code> - <message>true</message> - <threadName>false</threadName> - <dataType>true</dataType> - <encoding>false</encoding> - <assertions>false</assertions> - <subresults>false</subresults> - <responseData>true</responseData> - <samplerData>true</samplerData> - <xml>true</xml> - <fieldNames>true</fieldNames> - <responseHeaders>true</responseHeaders> - <requestHeaders>true</requestHeaders> - <responseDataOnError>true</responseDataOnError> - <saveAssertionResultsFailureMessage>false</saveAssertionResultsFailureMessage> - <assertionsResultsToSave>0</assertionsResultsToSave> - <url>true</url> - <hostname>true</hostname> - <sampleCount>true</sampleCount> - </value> - </objProp> - <stringProp name="filename">/tmp/amsJmeterTestResults.jtl</stringProp> - <stringProp name="RespTimeGraph.interval">15000</stringProp> - <intProp name="RespTimeGraph.linestrockwidth">2</intProp> - <intProp name="RespTimeGraph.lineshapepoint">4</intProp> - </ResultCollector> - <hashTree/> - </hashTree> - </hashTree> -</jmeterTestPlan>