Repository: hive Updated Branches: refs/heads/master 6a82a898e -> bf98700a7
HIVE-16206: Make Codahale metrics reporters pluggable (Sunitha Beeram via Carl Steinbach) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bf98700a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bf98700a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bf98700a Branch: refs/heads/master Commit: bf98700a760ea1cca30f236e7dce10272fffdd48 Parents: 6a82a89 Author: Carl Steinbach <c...@apache.org> Authored: Mon Apr 3 10:09:53 2017 -0700 Committer: Carl Steinbach <c...@apache.org> Committed: Mon Apr 3 10:09:53 2017 -0700 ---------------------------------------------------------------------- .../metrics/metrics2/CodahaleMetrics.java | 192 +++++++++---------- .../metrics/metrics2/CodahaleReporter.java | 29 +++ .../metrics2/ConsoleMetricsReporter.java | 55 ++++++ .../metrics/metrics2/JmxMetricsReporter.java | 56 ++++++ .../metrics2/JsonFileMetricsReporter.java | 136 +++++++++++++ .../metrics/metrics2/Metrics2Reporter.java | 62 ++++++ .../org/apache/hadoop/hive/conf/HiveConf.java | 22 ++- .../metrics/metrics2/TestCodahaleMetrics.java | 7 +- .../metrics2/TestCodahaleReportersConf.java | 145 ++++++++++++++ 9 files changed, 589 insertions(+), 115 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java index e8abf6c..2d6c1b4 100644 --- a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleMetrics.java @@ -44,6 +44,8 @@ import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; import com.google.common.collect.Lists; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -190,22 +192,8 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co registerAll("threads", new ThreadStatesGaugeSet()); registerAll("classLoading", new ClassLoadingGaugeSet()); - //Metrics reporter - Set<MetricsReporting> finalReporterList = new HashSet<MetricsReporting>(); - List<String> metricsReporterNames = Lists.newArrayList( - Splitter.on(",").trimResults().omitEmptyStrings().split(conf.getVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER))); - - if(metricsReporterNames != null) { - for (String metricsReportingName : metricsReporterNames) { - try { - MetricsReporting reporter = MetricsReporting.valueOf(metricsReportingName.trim().toUpperCase()); - finalReporterList.add(reporter); - } catch (IllegalArgumentException e) { - LOGGER.warn("Metrics reporter skipped due to invalid configured reporter: " + metricsReportingName); - } - } - } - initReporting(finalReporterList); + //initialize reporters + initReporting(); } @@ -385,107 +373,99 @@ public class CodahaleMetrics implements org.apache.hadoop.hive.common.metrics.co } /** - * Should be only called once to initialize the reporters + * Initializes reporters from HIVE_CODAHALE_METRICS_REPORTER_CLASSES or HIVE_METRICS_REPORTER if the former is not defined. + * Note: if both confs are defined, only HIVE_CODAHALE_METRICS_REPORTER_CLASSES will be used. */ - private void initReporting(Set<MetricsReporting> reportingSet) { - for (MetricsReporting reporting : reportingSet) { - switch(reporting) { - case CONSOLE: - final ConsoleReporter consoleReporter = ConsoleReporter.forRegistry(metricRegistry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build(); - consoleReporter.start(1, TimeUnit.SECONDS); - reporters.add(consoleReporter); - break; - case JMX: - final JmxReporter jmxReporter = JmxReporter.forRegistry(metricRegistry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build(); - jmxReporter.start(); - reporters.add(jmxReporter); - break; - case JSON_FILE: - final JsonFileReporter jsonFileReporter = new JsonFileReporter(); - jsonFileReporter.start(); - reporters.add(jsonFileReporter); - break; - case HADOOP2: - String applicationName = conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_COMPONENT_NAME.varname); - long reportingInterval = HiveConf.toTime( - conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_INTERVAL.varname), - TimeUnit.SECONDS, TimeUnit.SECONDS); - final HadoopMetrics2Reporter metrics2Reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry) - .convertRatesTo(TimeUnit.SECONDS) - .convertDurationsTo(TimeUnit.MILLISECONDS) - .build(DefaultMetricsSystem.initialize(applicationName), // The application-level name - applicationName, // Component name - applicationName, // Component description - "General"); // Name for each metric record - metrics2Reporter.start(reportingInterval, TimeUnit.SECONDS); - break; - } + private void initReporting() { + + if (!(initCodahaleMetricsReporterClasses() || initMetricsReporter())) { + LOGGER.warn("Unable to initialize metrics reporting"); + } + if (reporters.isEmpty()) { + // log a warning incase no reporters were successfully added + LOGGER.warn("No reporters configured for codahale metrics!"); } } - class JsonFileReporter implements Closeable { - private ObjectMapper jsonMapper = null; - private java.util.Timer timer = null; - - public void start() { - this.jsonMapper = new ObjectMapper().registerModule(new MetricsModule(TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS, false)); - this.timer = new java.util.Timer(true); + /** + * Initializes reporting using HIVE_CODAHALE_METRICS_REPORTER_CLASSES. + * @return whether initialization was successful or not + */ + private boolean initCodahaleMetricsReporterClasses() { - long time = conf.getTimeVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, TimeUnit.MILLISECONDS); - final String pathString = conf.getVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION); + List<String> reporterClasses = Lists.newArrayList(Splitter.on(",").trimResults(). + omitEmptyStrings().split(conf.getVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES))); + if (reporterClasses.isEmpty()) { + return false; + } - timer.schedule(new TimerTask() { - @Override - public void run() { - BufferedWriter bw = null; - try { - String json = jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(metricRegistry); - Path tmpPath = new Path(pathString + ".tmp"); - URI tmpPathURI = tmpPath.toUri(); - FileSystem fs = null; - if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) { - //default local - fs = FileSystem.getLocal(conf); - } else { - fs = FileSystem.get(tmpPathURI, conf); - } - fs.delete(tmpPath, true); - bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true))); - bw.write(json); - bw.close(); - fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644)); - - Path path = new Path(pathString); - fs.rename(tmpPath, path); - fs.setPermission(path, FsPermission.createImmutable((short) 0644)); - } catch (Exception e) { - LOGGER.warn("Error writing JSON Metrics to file", e); - } finally { - try { - if (bw != null) { - bw.close(); - } - } catch (IOException e) { - //Ignore. - } - } + for (String reporterClass : reporterClasses) { + Class name = null; + try { + name = conf.getClassByName(reporterClass); + } catch (ClassNotFoundException e) { + LOGGER.error("Unable to instantiate metrics reporter class " + reporterClass + + " from conf HIVE_CODAHALE_METRICS_REPORTER_CLASSES", e); + throw new IllegalArgumentException(e); + } + try { + Constructor constructor = name.getConstructor(MetricRegistry.class, HiveConf.class); + CodahaleReporter reporter = (CodahaleReporter) constructor.newInstance(metricRegistry, conf); + reporter.start(); + reporters.add(reporter); + } catch (NoSuchMethodException | InstantiationException | + IllegalAccessException | InvocationTargetException e) { + LOGGER.error("Unable to instantiate using constructor(MetricRegistry, HiveConf) for" + + " reporter " + reporterClass + " from conf HIVE_CODAHALE_METRICS_REPORTER_CLASSES", + e); + throw new IllegalArgumentException(e); + } + } + return true; + } + /** + * Initializes reporting using HIVE_METRICS+REPORTER. + * @return whether initialization was successful or not + */ + private boolean initMetricsReporter() { - } - }, 0, time); + List<String> metricsReporterNames = Lists.newArrayList(Splitter.on(",").trimResults(). + omitEmptyStrings().split(conf.getVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER))); + if (metricsReporterNames.isEmpty()) { + return false; } - @Override - public void close() { - if (timer != null) { - this.timer.cancel(); + MetricsReporting reporter = null; + for (String metricsReportingName : metricsReporterNames) { + try { + reporter = MetricsReporting.valueOf(metricsReportingName.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + LOGGER.error("Invalid reporter name " + metricsReportingName, e); + throw e; + } + CodahaleReporter codahaleReporter = null; + switch (reporter) { + case CONSOLE: + codahaleReporter = new ConsoleMetricsReporter(metricRegistry, conf); + break; + case JMX: + codahaleReporter = new JmxMetricsReporter(metricRegistry, conf); + break; + case JSON_FILE: + codahaleReporter = new JsonFileMetricsReporter(metricRegistry, conf); + break; + case HADOOP2: + codahaleReporter = new Metrics2Reporter(metricRegistry, conf); + break; + default: + LOGGER.warn("Unhandled reporter " + reporter + " provided."); + } + if (codahaleReporter != null) { + codahaleReporter.start(); + reporters.add(codahaleReporter); } } + return true; } } http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleReporter.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleReporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleReporter.java new file mode 100644 index 0000000..ba4d8e4 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/CodahaleReporter.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.metrics.metrics2; + +import com.codahale.metrics.Reporter; +import java.io.Closeable; + +interface CodahaleReporter extends Closeable, Reporter { + + /** + * Start the reporter. + */ + public void start(); +} http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/ConsoleMetricsReporter.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/ConsoleMetricsReporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/ConsoleMetricsReporter.java new file mode 100644 index 0000000..dea1848 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/ConsoleMetricsReporter.java @@ -0,0 +1,55 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.metrics.metrics2; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Reporter; +import java.io.Closeable; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.HiveConf; + + +/** + * A wrapper around Codahale ConsoleReporter to make it a pluggable/configurable Hive Metrics reporter. + */ +public class ConsoleMetricsReporter implements CodahaleReporter { + + private final ConsoleReporter reporter; + + public ConsoleMetricsReporter(MetricRegistry registry, HiveConf conf) { + + reporter = ConsoleReporter.forRegistry(registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(); + + } + + @Override + public void start() { + reporter.start(1, TimeUnit.SECONDS); + } + + @Override + public void close() { + reporter.close(); + } +} + http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java new file mode 100644 index 0000000..f12adf9 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JmxMetricsReporter.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.metrics.metrics2; + +import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Reporter; +import java.io.Closeable; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.HiveConf; + +/** + * A wrapper around Codahale JmxReporter to make it a pluggable/configurable Hive Metrics reporter. + */ +public class JmxMetricsReporter implements CodahaleReporter { + + private final MetricRegistry registry; + private final HiveConf conf; + private final JmxReporter jmxReporter; + + public JmxMetricsReporter(MetricRegistry registry, HiveConf conf) { + this.registry = registry; + this.conf = conf; + + jmxReporter = JmxReporter.forRegistry(registry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(); + } + + @Override + public void start() { + jmxReporter.start(); + } + + @Override + public void close() { + jmxReporter.close(); + } + +} http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java new file mode 100644 index 0000000..c07517a --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/JsonFileMetricsReporter.java @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.common.metrics.metrics2; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.json.MetricsModule; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.URI; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hive.conf.HiveConf; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + + +/** + * A metrics reporter for CodahaleMetrics that dumps metrics periodically into a file in JSON format. + */ + +public class JsonFileMetricsReporter implements CodahaleReporter { + + private final MetricRegistry metricRegistry; + private final ObjectWriter jsonWriter; + private final ScheduledExecutorService executorService; + private final HiveConf conf; + private final long interval; + private final String pathString; + private final Path path; + + private static final Logger LOGGER = LoggerFactory.getLogger(JsonFileMetricsReporter.class); + + public JsonFileMetricsReporter(MetricRegistry registry, HiveConf conf) { + this.metricRegistry = registry; + this.jsonWriter = + new ObjectMapper().registerModule(new MetricsModule(TimeUnit.MILLISECONDS, + TimeUnit.MILLISECONDS, false)).writerWithDefaultPrettyPrinter(); + executorService = Executors.newSingleThreadScheduledExecutor(); + this.conf = conf; + + interval = conf.getTimeVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, TimeUnit.MILLISECONDS); + pathString = conf.getVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION); + path = new Path(pathString); + } + + @Override + public void start() { + + final Path tmpPath = new Path(pathString + ".tmp"); + URI tmpPathURI = tmpPath.toUri(); + final FileSystem fs; + try { + if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) { + //default local + fs = FileSystem.getLocal(conf); + } else { + fs = FileSystem.get(tmpPathURI, conf); + } + } + catch (IOException e) { + LOGGER.error("Unable to access filesystem for path " + tmpPath + ". Aborting reporting", e); + return; + } + + Runnable task = new Runnable() { + public void run() { + try { + String json = null; + try { + json = jsonWriter.writeValueAsString(metricRegistry); + } catch (JsonProcessingException e) { + LOGGER.error("Unable to convert json to string ", e); + return; + } + + BufferedWriter bw = null; + try { + fs.delete(tmpPath, true); + bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true))); + bw.write(json); + fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644)); + } catch (IOException e) { + LOGGER.error("Unable to write to temp file " + tmpPath, e); + return; + } finally { + if (bw != null) { + bw.close(); + } + } + + try { + fs.rename(tmpPath, path); + fs.setPermission(path, FsPermission.createImmutable((short) 0644)); + } catch (IOException e) { + LOGGER.error("Unable to rename temp file " + tmpPath + " to " + pathString, e); + return; + } + } catch (Throwable t) { + // catch all errors (throwable and execptions to prevent subsequent tasks from being suppressed) + LOGGER.error("Error executing scheduled task ", t); + } + } + }; + + executorService.scheduleWithFixedDelay(task,0, interval, TimeUnit.MILLISECONDS); + } + + @Override + public void close() { + executorService.shutdown(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java new file mode 100644 index 0000000..3b402d8 --- /dev/null +++ b/common/src/java/org/apache/hadoop/hive/common/metrics/metrics2/Metrics2Reporter.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.metrics.metrics2; + +import com.codahale.metrics.MetricRegistry; +import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter; +import java.io.Closeable; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import com.codahale.metrics.Reporter; + +/** + * A wrapper around Codahale HadoopMetrics2Reporter to make it a pluggable/configurable Hive Metrics reporter. + */ +public class Metrics2Reporter implements CodahaleReporter { + + private final MetricRegistry metricRegistry; + private final HiveConf conf; + private final HadoopMetrics2Reporter reporter; + + public Metrics2Reporter(MetricRegistry registry, HiveConf conf) { + this.metricRegistry = registry; + this.conf = conf; + String applicationName = conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_COMPONENT_NAME.varname); + + reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry) + .convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS) + .build(DefaultMetricsSystem.initialize(applicationName), // The application-level name + applicationName, // Component name + applicationName, // Component description + "General"); // Name for each metric record + } + + @Override + public void start() { + long reportingInterval = + HiveConf.toTime(conf.get(HiveConf.ConfVars.HIVE_METRICS_HADOOP2_INTERVAL.varname), TimeUnit.SECONDS, TimeUnit.SECONDS); + reporter.start(reportingInterval, TimeUnit.SECONDS); + } + + @Override + public void close() { + reporter.close(); + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index b9a72d1..6de6ed6 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -2210,19 +2210,29 @@ public class HiveConf extends Configuration { "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics", "org.apache.hadoop.hive.common.metrics.LegacyMetrics"), "Hive metrics subsystem implementation class."), - HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "JSON_FILE, JMX", - "Reporter type for metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics, " + - "comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"), + HIVE_CODAHALE_METRICS_REPORTER_CLASSES("hive.service.metrics.codahale.reporter.classes", + "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " + + "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter", + "Comma separated list of reporter implementation classes for metric class " + + "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics. Overrides " + + "HIVE_METRICS_REPORTER conf if present"), + @Deprecated + HIVE_METRICS_REPORTER("hive.service.metrics.reporter", "", + "Reporter implementations for metric class " + + "org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics;" + + "Deprecated, use HIVE_CODAHALE_METRICS_REPORTER_CLASSES instead. This configuraiton will be" + + " overridden by HIVE_CODAHALE_METRICS_REPORTER_CLASSES if present. " + + "Comma separated list of JMX, CONSOLE, JSON_FILE, HADOOP2"), HIVE_METRICS_JSON_FILE_LOCATION("hive.service.metrics.file.location", "/tmp/report.json", "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, the location of local JSON metrics file. " + "This file will get overwritten at every interval."), - HIVE_METRICS_JSON_FILE_INTERVAL("hive.service.metrics.file.frequency", "5s", + HIVE_METRICS_JSON_FILE_INTERVAL("hive.service.metrics.file.frequency", "5000ms", new TimeValidator(TimeUnit.MILLISECONDS), - "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics JSON_FILE reporter, " + + "For metric class org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " + "the frequency of updating JSON metrics file."), HIVE_METRICS_HADOOP2_INTERVAL("hive.service.metrics.hadoop2.frequency", "30s", new TimeValidator(TimeUnit.SECONDS), - "For metric class org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics HADOOP2 reporter, " + + "For metric class org.apache.hadoop.hive.common.metrics.metrics2.Metrics2Reporter, " + "the frequency of updating the HADOOP2 metrics system."), HIVE_METRICS_HADOOP2_COMPONENT_NAME("hive.service.metrics.hadoop2.component", "hive", http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java index aa4e75f..67f81d6 100644 --- a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleMetrics.java @@ -33,8 +33,6 @@ import org.junit.Before; import org.junit.Test; import java.io.File; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -57,9 +55,12 @@ public class TestCodahaleMetrics { jsonReportFile = new File(workDir, "json_reporting"); jsonReportFile.delete(); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); - conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, MetricsReporting.JSON_FILE.name() + "," + MetricsReporting.JMX.name()); + conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES, + "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " + + "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter"); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); http://git-wip-us.apache.org/repos/asf/hive/blob/bf98700a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java ---------------------------------------------------------------------- diff --git a/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java new file mode 100644 index 0000000..d694e26 --- /dev/null +++ b/common/src/test/org/apache/hadoop/hive/common/metrics/metrics2/TestCodahaleReportersConf.java @@ -0,0 +1,145 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.common.metrics.metrics2; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.lang.reflect.InvocationTargetException; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.hive.common.metrics.MetricsTestUtils; +import org.apache.hadoop.hive.common.metrics.common.MetricsFactory; +import org.apache.hadoop.hive.conf.HiveConf; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; + +/** + * Unit tests for Codahale reporter config backward compatibility + */ +public class TestCodahaleReportersConf { + + private static File workDir = new File(System.getProperty("test.tmp.dir")); + private static File jsonReportFile; + + @After + public void after() throws Exception { + MetricsFactory.close(); + } + + /** + * Tests that the deprecated HIVE_METRICS_REPORTER config is used if the HIVE_CODAHALE_METRICS_REPORTER_CLASSES is missing. + */ + @Test + public void testFallbackToDeprecatedConfig() throws Exception { + + HiveConf conf = new HiveConf(); + + jsonReportFile = new File(workDir, "json_reporting"); + jsonReportFile.delete(); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + + MetricsFactory.init(conf); + + int runs = 5; + for (int i = 0; i < runs; i++) { + MetricsFactory.getInstance().incrementCounter("count2"); + } + + // we expect json file to be updated + byte[] jsonData = MetricsTestUtils.getFileData(jsonReportFile.getAbsolutePath(), 2000, 3); + ObjectMapper objectMapper = new ObjectMapper(); + + JsonNode rootNode = objectMapper.readTree(jsonData); + JsonNode countersNode = rootNode.path("counters"); + JsonNode methodCounterNode = countersNode.path("count2"); + JsonNode countNode = methodCounterNode.path("count"); + Assert.assertEquals(countNode.asInt(), 5); + } + + /** + * Tests that the deprecated HIVE_METRICS_REPORTER config is not used if + * HIVE_CODAHALE_METRICS_REPORTER_CLASSES is present. + * + * The deprecated config specifies json reporters whereas the newer one doesn't. Validates that + * the JSON file is not created. + */ + @Test + public void testNoFallback() throws Exception { + + HiveConf conf = new HiveConf(); + + jsonReportFile = new File(workDir, "json_reporting"); + jsonReportFile.delete(); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON"); + conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES, + "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + + MetricsFactory.init(conf); + + int runs = 5; + for (int i = 0; i < runs; i++) { + MetricsFactory.getInstance().incrementCounter("count2"); + } + + Assert.assertFalse(jsonReportFile.exists()); + } + + /** + * Tests that the deprecated HIVE_METRICS_REPORTER config is not used if + * HIVE_CODAHALE_METRICS_REPORTER_CLASSES is present but incorrect. + * + * The deprecated config specifies json reporters whereas the newer one doesn't. Validates that + * the JSON file is not created. + */ + @Test + public void testNoFallbackOnIncorrectConf() throws Exception { + + HiveConf conf = new HiveConf(); + + jsonReportFile = new File(workDir, "json_reporting"); + jsonReportFile.delete(); + + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, "local"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_CLASS, CodahaleMetrics.class.getCanonicalName()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_REPORTER, "JMX, JSON"); + conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES, + "org.apache.hadoop.hive.common.metrics.NonExistentReporter"); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_LOCATION, jsonReportFile.toString()); + conf.setVar(HiveConf.ConfVars.HIVE_METRICS_JSON_FILE_INTERVAL, "100ms"); + + try { + MetricsFactory.init(conf); + } catch (InvocationTargetException expectedException) { + + } + + Assert.assertFalse(jsonReportFile.exists()); + } +}