AMBARI-19879. Updating yarn-env and hadoop-env templates with ZK secure options on stack upgrade (Attila Magyar via magyari_sandor)
(cherry picked from commit 8ef31458a989dec5fb2b7f35223dd689fdeaba7b) Change-Id: I4f77c3631df581e1d9255bdce16f6ebcaeed029f Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/47d94bf5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/47d94bf5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/47d94bf5 Branch: refs/heads/branch-feature-BUG-74026 Commit: 47d94bf5ed3d8025b5041f6d2cbc3444ed529d3b Parents: 664c3ba Author: Attila Magyar <amag...@hortonworks.com> Authored: Tue Feb 7 14:42:03 2017 +0100 Committer: Zuul <rel...@hortonworks.com> Committed: Tue Feb 7 10:29:04 2017 -0800 ---------------------------------------------------------------------- .../HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py | 3 +-- .../stacks/HDP/2.3/upgrades/config-upgrade.xml | 9 +++++++++ .../HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml | 12 ++++++++++++ .../resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml | 2 ++ .../stacks/HDP/2.4/upgrades/config-upgrade.xml | 8 ++++++++ .../HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml | 14 ++++++++++++++ .../resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml | 2 ++ .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 8 ++++++++ .../HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml | 13 +++++++++++++ .../resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml | 2 ++ 10 files changed, 71 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py index 275d349..74b72c7 100644 --- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py +++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/zkfc_slave.py @@ -39,8 +39,6 @@ from resource_management.libraries.functions.stack_features import check_stack_f from resource_management.libraries.script import Script from resource_management.core.resources.zkmigrator import ZkMigrator - - class ZkfcSlave(Script): def get_component_name(self): import params @@ -62,6 +60,7 @@ class ZkfcSlave(Script): import params env.set_params(params) hdfs("zkfc_slave") + utils.set_up_zkfc_security(params) pass @OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT) http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml index f3e8ddd..f86b03d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/config-upgrade.xml @@ -375,6 +375,11 @@ <regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/> <regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/> </definition> + + <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs"> + <type>hadoop-env</type> + <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> + </definition> </changes> </component> </service> @@ -504,6 +509,10 @@ <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/> <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/> </definition> + <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs"> + <type>yarn-env</type> + <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml index dc53c67..8b7451e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.6.xml @@ -286,6 +286,12 @@ </task> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs"> + <task xsi:type="configure" id="hadoop_env_zkfc_security_opts"> + <summary>Adding HDFS ZKFC Security ACLs</summary> + </task> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig"> @@ -293,6 +299,12 @@ </task> </execute-stage> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs"> + <task xsi:type="configure" id="yarn_env_security_opts"> + <summary>Adding YARN Security ACLs</summary> + </task> + </execute-stage> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Apply config changes for Yarn Resourcemanager"> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/> </execute-stage> http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml index 290d3c5..01fc102 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.6.xml @@ -680,6 +680,7 @@ <task xsi:type="configure" id="hdp_2_4_0_0_namenode_ha_adjustments"/> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" /> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> + <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -760,6 +761,7 @@ </task> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" /> <task xsi:type="configure" id="yarn_log4j_parameterize" /> + <task xsi:type="configure" id="yarn_env_security_opts" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml index 6a462ec..57227f6 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/config-upgrade.xml @@ -264,6 +264,10 @@ <regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/> <regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/> </definition> + <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs"> + <type>hadoop-env</type> + <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> + </definition> </changes> </component> </service> @@ -289,6 +293,10 @@ <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/> <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/> </definition> + <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs"> + <type>yarn-env</type> + <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> + </definition> </changes> </component> http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml index 8a988cc..5661641 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.6.xml @@ -281,6 +281,13 @@ </task> </execute-stage> + <!--Yarn--> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs"> + <task xsi:type="configure" id="yarn_env_security_opts"> + <summary>Adding YARN Security ACLs</summary> + </task> + </execute-stage> + <!-- YARN --> <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle"> <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/> @@ -400,6 +407,13 @@ </task> </execute-stage> + <!--HDFS--> + <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs"> + <task xsi:type="configure" id="hadoop_env_zkfc_security_opts"> + <summary>Adding HDFS ZKFC Security ACLs</summary> + </task> + </execute-stage> + <!-- SQOOP --> <execute-stage service="SQOOP" component="SQOOP" title="Apply config changes for Sqoop to remove Atlas Configs"> <!-- Remove Atlas configs that were incorrectly added to sqoop-site instead of Atlas' application.properties. --> http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml index cadef73..7b12af5 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.6.xml @@ -685,6 +685,7 @@ <task xsi:type="configure" id="hdp_2_5_0_0_namenode_ha_adjustments"/> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_hdfs_audit_db" /> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> + <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> @@ -765,6 +766,7 @@ </task> <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db" /> <task xsi:type="configure" id="yarn_log4j_parameterize" /> + <task xsi:type="configure" id="yarn_env_security_opts" /> </pre-upgrade> <pre-downgrade /> <!-- no-op to prevent config changes on downgrade --> http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 188e6f4..54a824d 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -128,6 +128,10 @@ <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxFileSize=([0-9]+)MB" replace-with="log4j.appender.RMSUMMARY.MaxFileSize={{yarn_rm_summary_log_max_backup_size}}MB"/> <regex-replace key="content" find="^log4j.appender.RMSUMMARY.MaxBackupIndex=([0-9]+)" replace-with="log4j.appender.RMSUMMARY.MaxBackupIndex={{yarn_rm_summary_log_number_of_backup_files}}"/> </definition> + <definition xsi:type="configure" id="yarn_env_security_opts" summary="Adding YARN Security ACLs"> + <type>yarn-env</type> + <insert key="content" value="{% if rm_security_opts is defined %} YARN_OPTS="{{rm_security_opts}} $YARN_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> + </definition> </changes> </component> </service> @@ -146,6 +150,10 @@ <regex-replace key="content" find="hadoop.security.log.maxfilesize=([0-9]+)MB" replace-with="hadoop.security.log.maxfilesize={{hadoop_security_log_max_backup_size}}MB"/> <regex-replace key="content" find="hadoop.security.log.maxbackupindex=([0-9]+)" replace-with="hadoop.security.log.maxbackupindex={{hadoop_security_log_number_of_backup_files}}"/> </definition> + <definition xsi:type="configure" id="hadoop_env_zkfc_security_opts" summary="Adding HDFS ZKFC Security ACLs"> + <type>hadoop-env</type> + <insert key="content" value="{% if hadoop_zkfc_opts is defined %} export HADOOP_ZKFC_OPTS="{{hadoop_zkfc_opts}} $HADOOP_ZKFC_OPTS" {% endif %}" insert-type="append" newline-before="true" newline-after="true" /> + </definition> </changes> </component> </service> http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml index 1fad885..7bb679e 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/nonrolling-upgrade-2.6.xml @@ -291,6 +291,13 @@ </task> </execute-stage> + <!--Yarn--> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Adding YARN Security ACLs"> + <task xsi:type="configure" id="yarn_env_security_opts"> + <summary>Adding YARN Security ACLs</summary> + </task> + </execute-stage> + <!--TEZ--> <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez"> <task xsi:type="server_action" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"> @@ -317,6 +324,12 @@ </task> </execute-stage> + <execute-stage service="HDFS" component="NAMENODE" title="Adding HDFS ZKFC Security ACLs"> + <task xsi:type="configure" id="hadoop_env_zkfc_security_opts"> + <summary>Adding HDFS ZKFC Security ACLs</summary> + </task> + </execute-stage> + <!--HBASE--> <execute-stage service="HBASE" component="HBASE_MASTER" title="Parameterizing HBase Log4J Properties"> <task xsi:type="configure" id="hbase_log4j_parameterize"> http://git-wip-us.apache.org/repos/asf/ambari/blob/47d94bf5/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml index 291397a..2f07c97 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/upgrade-2.6.xml @@ -589,6 +589,7 @@ <component name="NAMENODE"> <pre-upgrade> <task xsi:type="configure" id="hdfs_log4j_parameterize" /> + <task xsi:type="configure" id="hadoop_env_zkfc_security_opts" /> </pre-upgrade> <pre-downgrade /> <upgrade> @@ -667,6 +668,7 @@ <component name="RESOURCEMANAGER"> <pre-upgrade> <task xsi:type="configure" id="yarn_log4j_parameterize" /> + <task xsi:type="configure" id="yarn_env_security_opts" /> </pre-upgrade> <pre-downgrade /> <upgrade>