Repository: ambari Updated Branches: refs/heads/trunk 46b2fcdef -> f8e83d365
AMBARI-16857. Support yarn aux service classpath isolation feature in spark (Bikas Saha via smohanty) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f8e83d36 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f8e83d36 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f8e83d36 Branch: refs/heads/trunk Commit: f8e83d365a87eb1651cfc7a5a1e48b7df6e5e6bf Parents: 46b2fcd Author: Sumit Mohanty <smoha...@hortonworks.com> Authored: Mon May 30 17:45:02 2016 -0700 Committer: Sumit Mohanty <smoha...@hortonworks.com> Committed: Mon May 30 17:45:02 2016 -0700 ---------------------------------------------------------------------- .../HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml | 4 ++ .../stacks/HDP/2.3/upgrades/upgrade-2.5.xml | 3 ++ .../HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml | 5 +++ .../stacks/HDP/2.4/upgrades/upgrade-2.5.xml | 3 ++ .../services/YARN/configuration/yarn-site.xml | 43 ++++++++++++++++++++ .../stacks/HDP/2.5/upgrades/config-upgrade.xml | 11 +++++ 6 files changed, 69 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml index 4362fe0..c7d8b30 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.5.xml @@ -291,6 +291,10 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/> </execute-stage> + <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle"> + <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/> + </execute-stage> + <execute-stage service="MAPREDUCE2" component="MAPREDUCE2_CLIENT" title="Apply config changes for Mapreduce2 client"> <task xsi:type="server_action" summary="Verifying LZO codec path for mapreduce" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/> </execute-stage> http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml index 0e0ea1d..e39f413 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.5.xml @@ -590,6 +590,9 @@ </component> <component name="NODEMANAGER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/> + </pre-upgrade> <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml index 5ccb153..47e849c 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/nonrolling-upgrade-2.5.xml @@ -277,6 +277,11 @@ <task xsi:type="configure" id="hdp_2_5_0_0_remove_ranger_yarn_audit_db"/> </execute-stage> + <!-- YARN --> + <execute-stage service="YARN" component="NODEMANAGER" title="Add Spark2 shuffle"> + <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/> + </execute-stage> + <!--TEZ--> <execute-stage service="TEZ" component="TEZ_CLIENT" title="Verify LZO codec path for Tez"> <task xsi:type="server_action" summary="Verifying LZO codec path for Tez" class="org.apache.ambari.server.serveraction.upgrades.FixLzoCodecPath"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml index bc45be6..644ca87 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.4/upgrades/upgrade-2.5.xml @@ -584,6 +584,9 @@ </component> <component name="NODEMANAGER"> + <pre-upgrade> + <task xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"/> + </pre-upgrade> <upgrade> <task xsi:type="restart-task" /> </upgrade> http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml new file mode 100644 index 0000000..0f0e44f --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/YARN/configuration/yarn-site.xml @@ -0,0 +1,43 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration supports_final="true"> + <property> + <name>yarn.nodemanager.aux-services</name> + <value>mapreduce_shuffle,spark_shuffle,spark2_shuffle</value> + <description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can + not start with numbers</description> + </property> + + <property> + <name>yarn.nodemanager.aux-services.spark2_shuffle.class</name> + <value>org.apache.spark.network.yarn.YarnShuffleService</value> + <description>The auxiliary service class to use for Spark 2</description> + </property> + + <property> + <name>yarn.nodemanager.aux-services.spark_shuffle.classpath</name> + <value>/usr/hdp/${hdp.version}/spark/aux/*</value> + </property> + + <property> + <name>yarn.nodemanager.aux-services.spark2_shuffle.classpath</name> + <value>/usr/hdp/${hdp.version}/spark2/aux/*</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/f8e83d36/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml index 5211276..aaa2d21 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/upgrades/config-upgrade.xml @@ -50,6 +50,17 @@ </changes> </component> </service> + <service name="YARN"> + <component name="NODEMANAGER"> + <changes> + <definition xsi:type="configure" id="hdp_2_5_0_0_add_spark2_yarn_shuffle"> + <type>yarn-site</type> + <set key="yarn.nodemanager.aux-services" value="mapreduce_shuffle,spark_shuffle,spark2_shuffle"/> + <!-- Ideally we need to append spark2_shuffle to the existing value --> + </definition> + </changes> + </component> + </service> </services> </upgrade-config-changes>
