AMBARI-14669. Add yarn-site properties for Spark Shuffle Aux services (2.2->2.4 upgrade path). (Dmytro Grinenko via mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e0957d39 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e0957d39 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e0957d39 Branch: refs/heads/branch-dev-patch-upgrade Commit: e0957d397a2780683cc3387c30359c68669a9343 Parents: 672fb60 Author: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Authored: Fri Jan 15 16:24:56 2016 +0200 Committer: Myroslav Papirkovskyy <mpapyrkovs...@hortonworks.com> Committed: Fri Jan 15 16:25:41 2016 +0200 ---------------------------------------------------------------------- .../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 4 + .../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 1 + .../SparkShufflePropertyConfigTest.java | 169 +++++++++++++++++++ 3 files changed, 174 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml index 1efeb6c..00b9e96 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml @@ -395,6 +395,10 @@ <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/> </execute-stage> + <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark"> + <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" /> + </execute-stage> + <!--HBASE--> <execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master"> <task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml index acebdd8..7637830 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml @@ -566,6 +566,7 @@ <task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/> + <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" /> </pre-upgrade> <upgrade> <task xsi:type="restart-task"/> http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java new file mode 100644 index 0000000..edba6f0 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.serveraction.upgrades; + +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.server.ServiceNotFoundException; +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.Config; +import org.apache.ambari.server.state.ConfigImpl; +import org.apache.ambari.server.state.Service; +import org.easymock.EasyMock; +import org.junit.Before; +import org.junit.Test; + +import com.google.inject.Injector; + +/** + * Tests upgrade-related server side actions for SparkShufflePropertyConfig + */ +public class SparkShufflePropertyConfigTest { + private Injector m_injector; + private Clusters m_clusters; + private Cluster cluster; + private Field clusterField; + + @Before + public void setup() throws Exception { + m_injector = EasyMock.createMock(Injector.class); + m_clusters = EasyMock.createMock(Clusters.class); + cluster = EasyMock.createMock(Cluster.class); + + + Config adminConfig = new ConfigImpl("yarn-site") { + Map<String, String> mockProperties = new HashMap<String, String>() {{ + put("yarn.nodemanager.aux-services", "some_service"); + }}; + @Override + public Map<String, String> getProperties() { + return mockProperties; + } + + @Override + public void setProperties(Map<String, String> properties) { + mockProperties.putAll(properties); + } + + @Override + public void persist(boolean newConfig) { + // no-op + } + }; + + expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(adminConfig).atLeastOnce(); + + expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes(); + expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce(); + + replay(m_injector, m_clusters); + + clusterField = SparkShufflePropertyConfig.class.getDeclaredField("clusters"); + clusterField.setAccessible(true); + } + + @Test + public void testSparkPresent() throws Exception { + ExecutionCommand executionCommand = new ExecutionCommand(); + Map<String, String> commandParams = new HashMap<>(); + HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); + Service sparkService = EasyMock.createMock(Service.class); + SparkShufflePropertyConfig action = new SparkShufflePropertyConfig(); + + commandParams.put("clusterName", "c1"); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + + // test calculations when spark service present + expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce(); + + expect(hrc.getRequestId()).andReturn(1L).anyTimes(); + expect(hrc.getStageId()).andReturn(2L).anyTimes(); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); + replay(cluster, hrc); + + clusterField.set(action, m_clusters); + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + CommandReport report = action.execute(null); + assertNotNull(report); + + Cluster c = m_clusters.getCluster("c1"); + Config config = c.getDesiredConfigByType("yarn-site"); + Map<String, String> map = config.getProperties(); + + assertTrue(map.containsKey("yarn.nodemanager.aux-services")); + assertTrue(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class")); + + assertEquals("some_service,spark_shuffle", map.get("yarn.nodemanager.aux-services")); + assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class")); + } + + @Test + public void testSparkNotPresent() throws Exception{ + SparkShufflePropertyConfig action = new SparkShufflePropertyConfig(); + Map<String, String> commandParams = new HashMap<>(); + ExecutionCommand executionCommand = new ExecutionCommand(); + HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class); + ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK"); + + commandParams.put("clusterName", "c1"); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName("c1"); + + //no spark service present + expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce(); + + expect(hrc.getRequestId()).andReturn(1L).anyTimes(); + expect(hrc.getStageId()).andReturn(2L).anyTimes(); + expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes(); + replay(cluster, hrc); + + clusterField.set(action, m_clusters); + action.setExecutionCommand(executionCommand); + action.setHostRoleCommand(hrc); + + CommandReport report = action.execute(null); + assertNotNull(report); + + Cluster c = m_clusters.getCluster("c1"); + Config config = c.getDesiredConfigByType("yarn-site"); + Map<String, String> map = config.getProperties(); + + assertTrue(map.containsKey("yarn.nodemanager.aux-services")); + assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class")); + assertEquals("some_service", map.get("yarn.nodemanager.aux-services")); + } +}
