Repository: hive Updated Branches: refs/heads/branch-2.0 82d784d14 -> 5333268b4
HIVE-13396: LLAP: Include hadoop-metrics2.properties file LlapServiceDriver (Prasanth Jayachandran reviewed by Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5333268b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5333268b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5333268b Branch: refs/heads/branch-2.0 Commit: 5333268b4140f08a69f7849f00011017cd956cba Parents: 82d784d Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Mon Apr 4 19:15:07 2016 -0500 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Tue Apr 5 13:39:16 2016 -0500 ---------------------------------------------------------------------- .../hadoop/hive/llap/cli/LlapServiceDriver.java | 19 +++++--- .../hive/llap/daemon/impl/LlapDaemon.java | 1 + .../llap/metrics/LlapDaemonCacheMetrics.java | 4 +- .../llap/metrics/LlapDaemonExecutorMetrics.java | 4 +- .../llap/metrics/LlapDaemonQueueMetrics.java | 4 +- .../hadoop/hive/llap/metrics/MetricsUtils.java | 1 - .../hadoop-metrics2.properties.template | 50 ++++++++++++++++++++ 7 files changed, 69 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/5333268b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java index 68c6b29..691fb87 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/LlapServiceDriver.java @@ -304,13 +304,18 @@ public class LlapServiceDriver { IOUtils.copyBytes(loggerContent, lfs.create(new Path(confPath, "llap-daemon-log4j2.properties"), true), conf, true); - String java_home = System.getenv("JAVA_HOME"); - String jre_home = System.getProperty("java.home"); - if (java_home == null) { - java_home = jre_home; - } else if (!java_home.equals(jre_home)) { - LOG.warn("Java versions might not match : JAVA_HOME=%s,process jre=%s", - java_home, jre_home); + URL metrics2 = conf.getResource(LlapDaemon.HADOOP_METRICS2_PROPERTIES_FILE); + if (metrics2 != null) { + InputStream metrics2FileStream = metrics2.openStream(); + IOUtils.copyBytes(metrics2FileStream, + lfs.create(new Path(confPath, LlapDaemon.HADOOP_METRICS2_PROPERTIES_FILE), true), + conf, true); + } + + PrintWriter udfStream = + new PrintWriter(lfs.create(new Path(confPath, StaticPermanentFunctionChecker.PERMANENT_FUNCTIONS_LIST))); + for (String udfClass : allowedUdfs) { + udfStream.println(udfClass); } // extract configs for processing by the python fragments in Slider http://git-wip-us.apache.org/repos/asf/hive/blob/5333268b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java index 9b7c163..be5fbd6 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/daemon/impl/LlapDaemon.java @@ -67,6 +67,7 @@ public class LlapDaemon extends CompositeService implements ContainerRunner, Lla private static final Logger LOG = LoggerFactory.getLogger(LlapDaemon.class); public static final String LOG4j2_PROPERTIES_FILE = "llap-daemon-log4j2.properties"; + public static final String HADOOP_METRICS2_PROPERTIES_FILE = "hadoop-metrics2.properties"; private final Configuration shuffleHandlerConf; private final LlapProtocolServerImpl server; private final ContainerRunnerImpl containerRunner; http://git-wip-us.apache.org/repos/asf/hive/blob/5333268b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java index 52057e4..b89c6c4 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonCacheMetrics.java @@ -45,7 +45,7 @@ import com.google.common.annotations.VisibleForTesting; /** * Llap daemon cache metrics source. */ -@Metrics(about = "LlapDaemon Cache Metrics", context = MetricsUtils.METRICS_CONTEXT) +@Metrics(about = "LlapDaemon Cache Metrics", context = "cache") public class LlapDaemonCacheMetrics implements MetricsSource { final String name; private String sessionId; @@ -127,7 +127,7 @@ public class LlapDaemonCacheMetrics implements MetricsSource { @Override public void getMetrics(MetricsCollector collector, boolean b) { MetricsRecordBuilder rb = collector.addRecord(CacheMetrics) - .setContext(MetricsUtils.METRICS_CONTEXT) + .setContext("cache") .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME) .tag(SessionId, sessionId); getCacheStats(rb); http://git-wip-us.apache.org/repos/asf/hive/blob/5333268b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java index 33b8f9d..7919200 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonExecutorMetrics.java @@ -52,7 +52,7 @@ import org.apache.hadoop.metrics2.source.JvmMetrics; /** * Metrics about the llap daemon executors. */ -@Metrics(about = "LlapDaemon Executor Metrics", context = MetricsUtils.METRICS_CONTEXT) +@Metrics(about = "LlapDaemon Executor Metrics", context = "executors") public class LlapDaemonExecutorMetrics implements MetricsSource { private final String name; @@ -117,7 +117,7 @@ public class LlapDaemonExecutorMetrics implements MetricsSource { @Override public void getMetrics(MetricsCollector collector, boolean b) { MetricsRecordBuilder rb = collector.addRecord(ExecutorMetrics) - .setContext(MetricsUtils.METRICS_CONTEXT) + .setContext("executors") .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME) .tag(SessionId, sessionId); getExecutorStats(rb); http://git-wip-us.apache.org/repos/asf/hive/blob/5333268b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java index 5905cae..10a0124 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/LlapDaemonQueueMetrics.java @@ -38,7 +38,7 @@ import org.apache.hadoop.metrics2.lib.MutableRate; /** * */ -@Metrics(about = "LlapDaemon Queue Metrics", context = MetricsUtils.METRICS_CONTEXT) +@Metrics(about = "LlapDaemon Queue Metrics", context = "queue") public class LlapDaemonQueueMetrics implements MetricsSource { private final String name; private final String sessionId; @@ -78,7 +78,7 @@ public class LlapDaemonQueueMetrics implements MetricsSource { @Override public void getMetrics(MetricsCollector collector, boolean b) { MetricsRecordBuilder rb = collector.addRecord(QueueMetrics) - .setContext(MetricsUtils.METRICS_CONTEXT) + .setContext("queue") .tag(ProcessName, MetricsUtils.METRICS_PROCESS_NAME) .tag(SessionId, sessionId); getQueueStats(rb); http://git-wip-us.apache.org/repos/asf/hive/blob/5333268b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java index ce1c965..c22ce4e 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/metrics/MetricsUtils.java @@ -26,7 +26,6 @@ import java.util.UUID; */ public class MetricsUtils { private static final String LOCALHOST = "localhost"; - public static final String METRICS_CONTEXT = "llap"; public static final String METRICS_PROCESS_NAME = "LlapDaemon"; http://git-wip-us.apache.org/repos/asf/hive/blob/5333268b/llap-server/src/main/resources/hadoop-metrics2.properties.template ---------------------------------------------------------------------- diff --git a/llap-server/src/main/resources/hadoop-metrics2.properties.template b/llap-server/src/main/resources/hadoop-metrics2.properties.template new file mode 100644 index 0000000..994acaa --- /dev/null +++ b/llap-server/src/main/resources/hadoop-metrics2.properties.template @@ -0,0 +1,50 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# syntax: [prefix].[source|sink].[instance].[options] +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details + +#*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink +# default sampling period, in seconds +#*.sink.file.period=10 + +# *.sink.timeline.class=org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink +# *.sink.timeline.period=60 + +# llapdeamon metrics for all contexts (jvm,queue,executors,cache) will go to this file +# llapdaemon.sink.file.filename=llapdaemon-metrics.out + +# to configure separate files per context define following for each context +# llapdaemon.sink.file_jvm.class=org.apache.hadoop.metrics2.sink.FileSink +# llapdaemon.sink.file_jvm.context=jvm +# llapdaemon.sink.file_jvm.filename=llapdaemon-jvm-metrics.out
