Repository: ambari Updated Branches: refs/heads/trunk 51c0cb4ff -> 46d1d7b18
AMBARI-16209. Write out new log property files under /etc/hive2/conf (smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/46d1d7b1 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/46d1d7b1 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/46d1d7b1 Branch: refs/heads/trunk Commit: 46d1d7b188d3f8c020f992c91406d994bac5eebc Parents: 51c0cb4 Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Tue May 3 07:30:40 2016 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Tue May 3 07:30:40 2016 -0700 ---------------------------------------------------------------------- .../package/scripts/hive_interactive.py | 29 +++-- .../0.12.0.2.0/package/scripts/params_linux.py | 5 + .../HIVE/configuration/llap-cli-log4j2.xml | 108 +++++++++++++++++++ .../HIVE/configuration/llap-daemon-log4j.xml | 21 +++- .../stacks/2.5/HIVE/test_hive_server_int.py | 12 +++ .../python/stacks/2.5/configs/hsi_default.json | 6 ++ 6 files changed, 167 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py index 24047ff..e8532da 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_interactive.py @@ -126,34 +126,43 @@ def hive_interactive(name=None): File(format("{hive_server_interactive_conf_dir}/hive-env.sh"), owner=params.hive_user, group=params.user_group, - content=InlineTemplate(params.hive_interactive_env_sh_template) - ) + content=InlineTemplate(params.hive_interactive_env_sh_template)) + + llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"), + mode=0644, + group=params.user_group, + owner=params.hive_user, + content=params.llap_daemon_log4j) + + llap_cli_log4j2_filename = 'llap-cli-log4j2.properties' + File(format("{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"), + mode=0644, + group=params.user_group, + owner=params.hive_user, + content=params.llap_cli_log4j2) # On some OS this folder could be not exists, so we will create it before pushing there files Directory(params.limits_conf_dir, create_parents = True, owner='root', - group='root' - ) + group='root') File(os.path.join(params.limits_conf_dir, 'hive.conf'), owner='root', group='root', mode=0644, - content=Template("hive.conf.j2") - ) + content=Template("hive.conf.j2")) if not os.path.exists(params.target_hive_interactive): jdbc_connector(params.target_hive_interactive) File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"), content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")), - mode = 0644, - ) + mode = 0644) File(params.start_hiveserver2_interactive_path, mode=0755, - content=Template(format('{start_hiveserver2_interactive_script}')) - ) + content=Template(format('{start_hiveserver2_interactive_script}'))) Directory(params.hive_pid_dir, create_parents=True, http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 2e1a2af..cef2cbc 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -538,12 +538,16 @@ HdfsResource = functools.partial( hive_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts', []) has_hive_interactive = len(hive_interactive_hosts) > 0 if has_hive_interactive: + llap_daemon_log4j = config['configurations']['llap-daemon-log4j']['content'] + llap_cli_log4j2 = config['configurations']['llap-cli-log4j2']['content'] + hive_server_interactive_conf_dir = status_params.hive_server_interactive_conf_dir execute_path_hive_interactive = os.path.join(os.environ['PATH'], hive_interactive_bin, hadoop_bin_dir) start_hiveserver2_interactive_script = 'startHiveserver2Interactive.sh.j2' start_hiveserver2_interactive_path = format("{tmp_dir}/start_hiveserver2_interactive_script") hive_interactive_env_sh_template = config['configurations']['hive-interactive-env']['content'] hive_interactive_enabled = default('/configurations/hive-interactive-env/enable_hive_interactive', False) + # Service check related if hive_transport_mode.lower() == "http": hive_server_interactive_port = config['configurations']['hive-interactive-site']['hive.server2.thrift.http.port'] @@ -563,6 +567,7 @@ if has_hive_interactive: if security_enabled: hive_llap_keytab_file = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.keytab.file'] hive_headless_keytab = config['configurations']['hive-interactive-site']['hive.llap.zk.sm.principal'] + pass # ranger host stack_supports_ranger_audit_db = stack_version_formatted_major and check_stack_feature(StackFeature.RANGER_AUDIT_DB_SUPPORT, stack_version_formatted_major) http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml new file mode 100644 index 0000000..5c8f9a8 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-cli-log4j2.xml @@ -0,0 +1,108 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> + +<configuration supports_final="false" supports_adding_forbidden="true"> + <property> + <name>content</name> + <description>Custom log4j.properties</description> + <value> +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +status = WARN +name = LlapCliLog4j2 +packages = org.apache.hadoop.hive.ql.log + +# list of properties +property.hive.log.level = INFO +property.hive.root.logger = console +property.hive.log.dir = ${sys:java.io.tmpdir}/${sys:user.name} +property.hive.log.file = llap-cli.log + +# list of all appenders +appenders = console, DRFA + +# console appender +appender.console.type = Console +appender.console.name = console +appender.console.target = SYSTEM_ERR +appender.console.layout.type = PatternLayout +# appender.console.layout.pattern = %d{yy/MM/dd HH:mm:ss} [%t]: %p %c{2}: %m%n +appender.console.layout.pattern = %p %c{2}: %m%n + +# daily rolling file appender +appender.DRFA.type = RollingFile +appender.DRFA.name = DRFA +appender.DRFA.fileName = ${sys:hive.log.dir}/${sys:hive.log.file} +# Use %pid in the filePattern to append <process-id>@<host-name> to the filename if you want separate log files for different CLI session +appender.DRFA.filePattern = ${sys:hive.log.dir}/${sys:hive.log.file}.%d{yyyy-MM-dd} +appender.DRFA.layout.type = PatternLayout +appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n +appender.DRFA.policies.type = Policies +appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy +appender.DRFA.policies.time.interval = 1 +appender.DRFA.policies.time.modulate = true +appender.DRFA.strategy.type = DefaultRolloverStrategy +appender.DRFA.strategy.max = 30 + +# list of all loggers +loggers = ZooKeeper, DataNucleus, Datastore, JPOX, HadoopConf + +logger.ZooKeeper.name = org.apache.zookeeper +logger.ZooKeeper.level = WARN + +logger.DataNucleus.name = DataNucleus +logger.DataNucleus.level = ERROR + +logger.Datastore.name = Datastore +logger.Datastore.level = ERROR + +logger.JPOX.name = JPOX +logger.JPOX.level = ERROR + +logger.HadoopConf.name = org.apache.hadoop.conf.Configuration +logger.HadoopConf.level = ERROR + +# root logger +rootLogger.level = ${sys:hive.log.level} +rootLogger.appenderRefs = root +rootLogger.appenderRef.root.ref = ${sys:hive.root.logger} + </value> + <value-attributes> + <type>content</type> + <show-property-name>false</show-property-name> + </value-attributes> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml index 1c60285..793ea7a 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/llap-daemon-log4j.xml @@ -40,12 +40,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -status = WARN + +status = INFO name = LlapDaemonLog4j2 packages = org.apache.hadoop.hive.ql.log # list of properties -property.llap.daemon.log.level = WARN +property.llap.daemon.log.level = INFO property.llap.daemon.root.logger = console property.llap.daemon.log.dir = . property.llap.daemon.log.file = llapdaemon.log @@ -90,7 +91,19 @@ appender.HISTORYAPPENDER.strategy.type = DefaultRolloverStrategy appender.HISTORYAPPENDER.strategy.max = ${sys:llap.daemon.log.maxbackupindex} # list of all loggers -loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger +loggers = NIOServerCnxn, ClientCnxnSocketNIO, DataNucleus, Datastore, JPOX, HistoryLogger, LlapIoImpl, LlapIoOrc, LlapIoCache, LlapIoLocking + +logger.LlapIoImpl.name = LlapIoImpl +logger.LlapIoImpl.level = INFO + +logger.LlapIoOrc.name = LlapIoOrc +logger.LlapIoOrc.level = WARN + +logger.LlapIoCache.name = LlapIoCache +logger.LlapIOCache.level = WARN + +logger.LlapIoLocking.name = LlapIoLocking +logger.LlapIoLocking.level = WARN logger.NIOServerCnxn.name = org.apache.zookeeper.server.NIOServerCnxn logger.NIOServerCnxn.level = WARN @@ -108,7 +121,7 @@ logger.JPOX.name = JPOX logger.JPOX.level = ERROR logger.HistoryLogger.name = org.apache.hadoop.hive.llap.daemon.HistoryLogger -logger.HistoryLogger.level = WARN +logger.HistoryLogger.level = INFO logger.HistoryLogger.additivity = false logger.HistoryLogger.appenderRefs = HistoryAppender logger.HistoryLogger.appenderRef.HistoryAppender.ref = HISTORYAPPENDER http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py index 5712995..6a0d777 100644 --- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py @@ -260,6 +260,18 @@ class TestHiveServerInteractive(RMFTestCase): owner='hive', group='hadoop', ) + self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/conf.server/llap-daemon-log4j2.properties', + content='con\ntent', + owner='hive', + group='hadoop', + mode=0644, + ) + self.assertResourceCalled('File', '/usr/hdp/current/hive-server2-hive2/conf/conf.server/llap-cli-log4j2.properties', + content='con\ntent', + owner='hive', + group='hadoop', + mode=0644, + ) self.assertResourceCalled('Directory', '/etc/security/limits.d', owner='root', group='root', http://git-wip-us.apache.org/repos/asf/ambari/blob/46d1d7b1/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json index e7bb056..9bff890 100644 --- a/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json +++ b/ambari-server/src/test/python/stacks/2.5/configs/hsi_default.json @@ -48,6 +48,12 @@ "taskId": 152, "public_hostname": "c6401.ambari.apache.org", "configurations": { + "llap-daemon-log4j": { + "content" : "con\ntent" + }, + "llap-cli-log4j2": { + "content" : "con\ntent" + }, "sqoop-site": { "atlas.cluster.name": "c1", "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook"
