AMBARI-19683. Content of yarn-env.sh on host is not same as in the downloaded config file from Ambari UI. (Attila Magyar via stoader)
(cherry picked from commit 99ec0a7f0a8dbb84455f68f86aa3ce3f9f761fee) Change-Id: I16e78a7d3fdb4e3939c5af20929a4bc4e9d91587 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/02984ffa Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/02984ffa Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/02984ffa Branch: refs/heads/branch-feature-BUG-74026 Commit: 02984ffa7c27e17d47a99960a6269737be3eada5 Parents: ea45fdd Author: Attila Magyar <amag...@hortonworks.com> Authored: Fri Feb 3 11:48:41 2017 +0100 Committer: Zuul <rel...@hortonworks.com> Committed: Tue Feb 7 01:29:02 2017 -0800 ---------------------------------------------------------------------- .../2.1.0.2.0/package/scripts/params_linux.py | 3 +- .../services/YARN/configuration/yarn-env.xml | 165 +++++++++++++++++++ .../stacks/2.0.6/YARN/test_historyserver.py | 3 +- .../stacks/2.0.6/YARN/test_mapreduce2_client.py | 3 +- .../stacks/2.0.6/YARN/test_nodemanager.py | 3 +- .../stacks/2.0.6/YARN/test_resourcemanager.py | 3 +- .../stacks/2.0.6/YARN/test_yarn_client.py | 3 +- 7 files changed, 172 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/02984ffa/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py index 5a47e03..f9228be 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py @@ -259,7 +259,8 @@ if security_enabled: rm_keytab = config['configurations']['yarn-site']['yarn.resourcemanager.keytab'] rm_kinit_cmd = format("{kinit_path_local} -kt {rm_keytab} {rm_principal_name};") yarn_jaas_file = os.path.join(config_dir, 'yarn_jaas.conf') - yarn_env_sh_template += format('\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config={yarn_jaas_file} -Dzookeeper.sasl.clientconfig=Client"\n') + if stack_supports_zk_security: + rm_security_opts = format('-Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config={yarn_jaas_file} -Dzookeeper.sasl.clientconfig=Client') # YARN timeline security options if has_ats: http://git-wip-us.apache.org/repos/asf/ambari/blob/02984ffa/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml new file mode 100644 index 0000000..d04c3c5 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.6/services/YARN/configuration/yarn-env.xml @@ -0,0 +1,165 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="true" supports_adding_forbidden="true"> + <property> + <name>is_supported_yarn_ranger</name> + <value>true</value> + <description>Set to false by default, needs to be set to true in stacks that use Ranger Yarn Plugin</description> + <on-ambari-upgrade add="true"/> + </property> + <!-- yarn-env.sh --> + <property> + <name>content</name> + <display-name>yarn-env template</display-name> + <description>This is the jinja template for yarn-env.sh file</description> + <value> + export HADOOP_YARN_HOME={{hadoop_yarn_home}} + export YARN_LOG_DIR={{yarn_log_dir_prefix}}/$USER + export YARN_PID_DIR={{yarn_pid_dir_prefix}}/$USER + export HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}} + export JAVA_HOME={{java64_home}} + export JAVA_LIBRARY_PATH="${JAVA_LIBRARY_PATH}:{{hadoop_java_io_tmpdir}}" + + # We need to add the EWMA appender for the yarn daemons only; + # however, YARN_ROOT_LOGGER is shared by the yarn client and the + # daemons. This is restrict the EWMA appender to daemons only. + INVOKER="${0##*/}" + if [ "$INVOKER" == "yarn-daemon.sh" ]; then + export YARN_ROOT_LOGGER=${YARN_ROOT_LOGGER:-INFO,EWMA,RFA} + fi + + # User for YARN daemons + export HADOOP_YARN_USER=${HADOOP_YARN_USER:-yarn} + + # resolve links - $0 may be a softlink + export YARN_CONF_DIR="${YARN_CONF_DIR:-$HADOOP_YARN_HOME/conf}" + + # some Java parameters + # export JAVA_HOME=/home/y/libexec/jdk1.6.0/ + if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME + fi + + if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 + fi + + JAVA=$JAVA_HOME/bin/java + JAVA_HEAP_MAX=-Xmx1000m + + # For setting YARN specific HEAP sizes please use this + # Parameter and set appropriately + YARN_HEAPSIZE={{yarn_heapsize}} + + # check envvars which might override default args + if [ "$YARN_HEAPSIZE" != "" ]; then + JAVA_HEAP_MAX="-Xmx""$YARN_HEAPSIZE""m" + fi + + # Resource Manager specific parameters + + # Specify the max Heapsize for the ResourceManager using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1000. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_RESOURCEMANAGER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_RESOURCEMANAGER_HEAPSIZE={{resourcemanager_heapsize}} + + # Specify the JVM options to be used when starting the ResourceManager. + # These options will be appended to the options specified as YARN_OPTS + # and therefore may override any similar flags set in YARN_OPTS + #export YARN_RESOURCEMANAGER_OPTS= + + # Node Manager specific parameters + + # Specify the max Heapsize for the NodeManager using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1000. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_NODEMANAGER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_NODEMANAGER_HEAPSIZE={{nodemanager_heapsize}} + + # Specify the max Heapsize for the timeline server using a numerical value + # in the scale of MB. For example, to specify an jvm option of -Xmx1000m, set + # the value to 1024. + # This value will be overridden by an Xmx setting specified in either YARN_OPTS + # and/or YARN_TIMELINESERVER_OPTS. + # If not specified, the default value will be picked from either YARN_HEAPMAX + # or JAVA_HEAP_MAX with YARN_HEAPMAX as the preferred option of the two. + export YARN_TIMELINESERVER_HEAPSIZE={{apptimelineserver_heapsize}} + + # Specify the JVM options to be used when starting the NodeManager. + # These options will be appended to the options specified as YARN_OPTS + # and therefore may override any similar flags set in YARN_OPTS + #export YARN_NODEMANAGER_OPTS= + + # so that filenames w/ spaces are handled correctly in loops below + IFS= + + + # default log directory and file + if [ "$YARN_LOG_DIR" = "" ]; then + YARN_LOG_DIR="$HADOOP_YARN_HOME/logs" + fi + if [ "$YARN_LOGFILE" = "" ]; then + YARN_LOGFILE='yarn.log' + fi + + # default policy file for service-level authorization + if [ "$YARN_POLICYFILE" = "" ]; then + YARN_POLICYFILE="hadoop-policy.xml" + fi + + # restore ordinary behaviour + unset IFS + + + YARN_OPTS="$YARN_OPTS -Dhadoop.log.dir=$YARN_LOG_DIR" + YARN_OPTS="$YARN_OPTS -Dyarn.log.dir=$YARN_LOG_DIR" + YARN_OPTS="$YARN_OPTS -Dhadoop.log.file=$YARN_LOGFILE" + YARN_OPTS="$YARN_OPTS -Dyarn.log.file=$YARN_LOGFILE" + YARN_OPTS="$YARN_OPTS -Dyarn.home.dir=$YARN_COMMON_HOME" + YARN_OPTS="$YARN_OPTS -Dyarn.id.str=$YARN_IDENT_STRING" + YARN_OPTS="$YARN_OPTS -Dhadoop.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" + YARN_OPTS="$YARN_OPTS -Dyarn.root.logger=${YARN_ROOT_LOGGER:-INFO,console}" + export YARN_NODEMANAGER_OPTS="$YARN_NODEMANAGER_OPTS -Dnm.audit.logger=INFO,NMAUDIT" + export YARN_RESOURCEMANAGER_OPTS="$YARN_RESOURCEMANAGER_OPTS -Drm.audit.logger=INFO,RMAUDIT" + if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + YARN_OPTS="$YARN_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" + fi + YARN_OPTS="$YARN_OPTS -Dyarn.policy.file=$YARN_POLICYFILE" + YARN_OPTS="$YARN_OPTS -Djava.io.tmpdir={{hadoop_java_io_tmpdir}}" + {% if rm_security_opts is defined %} + YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" + {% endif %} + </value> + <value-attributes> + <type>content</type> + </value-attributes> + <on-ambari-upgrade add="true"/> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/02984ffa/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py index 92e2e29..e5449d3 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_historyserver.py @@ -674,8 +674,7 @@ class TestHistoryServer(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + - '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), owner = 'yarn', group = 'hadoop', mode = 0755, http://git-wip-us.apache.org/repos/asf/ambari/blob/02984ffa/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py index 774f3c6..b05d9f2 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_mapreduce2_client.py @@ -305,8 +305,7 @@ class TestMapReduce2Client(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + - '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), owner = 'yarn', group = 'hadoop', mode = 0755, http://git-wip-us.apache.org/repos/asf/ambari/blob/02984ffa/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py index 0eb5561..10edb4b 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py @@ -487,8 +487,7 @@ class TestNodeManager(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + - '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), owner = 'yarn', group = 'hadoop', mode = 0755, http://git-wip-us.apache.org/repos/asf/ambari/blob/02984ffa/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py index da93096..dc40e5e 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py @@ -459,8 +459,7 @@ class TestResourceManager(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + - '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), owner = 'yarn', group = 'hadoop', mode = 0755, http://git-wip-us.apache.org/repos/asf/ambari/blob/02984ffa/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py index d4341e1..8873fbf 100644 --- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py +++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_yarn_client.py @@ -305,8 +305,7 @@ class TestYarnClient(RMFTestCase): mode = 0644, ) self.assertResourceCalled('File', '/etc/hadoop/conf/yarn-env.sh', - content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content'] + - '\nYARN_OPTS="$YARN_OPTS -Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username=zookeeper -Djava.security.auth.login.config=/usr/hdp/current/hadoop-client/conf/yarn_jaas.conf -Dzookeeper.sasl.clientconfig=Client"\n'), + content = InlineTemplate(self.getConfig()['configurations']['yarn-env']['content']), owner = 'yarn', group = 'hadoop', mode = 0755,