Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-20859 7460cebf9 -> 103dfffe7
AMBARI-21160 - Upgrade Execution Commands Are Missing Service/Component For Action Commands (jonathanhurley) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/36cc11b3 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/36cc11b3 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/36cc11b3 Branch: refs/heads/branch-feature-AMBARI-20859 Commit: 36cc11b3aa68fc93901ac6ed2fa2c838b3ee6400 Parents: 3912068 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Wed May 31 16:53:32 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Thu Jun 1 09:24:56 2017 -0400 ---------------------------------------------------------------------- .../internal/UpgradeResourceProvider.java | 26 +++++++--- .../0.12.0.2.0/package/scripts/params_linux.py | 2 +- .../2.1.0.3.0/package/scripts/params_linux.py | 2 +- .../AmbariManagementControllerTest.java | 6 +-- .../internal/UpgradeResourceProviderTest.java | 54 ++++++++++++++++++++ .../upgrades/upgrade_execute_task_test.xml | 42 +++++++++++++++ 6 files changed, 121 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/36cc11b3/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java index 345bf5f..f8f8faa 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java @@ -174,6 +174,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider */ public static final String UPGRADE_HOST_ORDERED_HOSTS = "Upgrade/host_order"; + /** + * The role that will be used when creating HRC's for the type + * {@link StageWrapper.Type#RU_TASKS}. + */ + protected static final String EXECUTE_TASK_ROLE = "ru_execute_tasks"; + /* * Lifted from RequestResourceProvider */ @@ -862,10 +868,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider Cluster cluster = context.getCluster(); - // add each host to this stage - RequestResourceFilter filter = new RequestResourceFilter("", "", - new ArrayList<>(wrapper.getHosts())); - LOG.debug("Analyzing upgrade item {} with tasks: {}.", entity.getText(), entity.getTasks()); Map<String, String> params = getNewParameterMap(request, context); params.put(UpgradeContext.COMMAND_PARAM_TASKS, entity.getTasks()); @@ -882,9 +884,17 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion()); + // if the service/component are specified, then make sure to grab them off + // of the wrapper so they can be stored on the command for use later + String serviceName = null; + String componentName = null; + if (wrapper.getTasks() != null && wrapper.getTasks().size() > 0 && wrapper.getTasks().get(0).getService() != null) { - String serviceName = wrapper.getTasks().get(0).getService(); + TaskWrapper taskWrapper = wrapper.getTasks().get(0); + serviceName = taskWrapper.getService(); + componentName = taskWrapper.getComponent(); + ServiceInfo serviceInfo = ambariMetaInfo.getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); @@ -892,8 +902,12 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider params.put(HOOKS_FOLDER, stackInfo.getStackHooksFolder()); } + // add each host to this stage + RequestResourceFilter filter = new RequestResourceFilter(serviceName, componentName, + new ArrayList<>(wrapper.getHosts())); + ActionExecutionContext actionContext = new ActionExecutionContext(cluster.getClusterName(), - "ru_execute_tasks", Collections.singletonList(filter), params); + EXECUTE_TASK_ROLE, Collections.singletonList(filter), params); // hosts in maintenance mode are excluded from the upgrade actionContext.setMaintenanceModeHostExcluded(true); http://git-wip-us.apache.org/repos/asf/ambari/blob/36cc11b3/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index c1128a5..f680799 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -399,7 +399,7 @@ hive_conf_dirs_list = [hive_client_conf_dir] ranger_hive_component = status_params.SERVER_ROLE_DIRECTORY_MAP['HIVE_SERVER'] if status_params.role == "HIVE_METASTORE" and hive_metastore_hosts is not None and hostname in hive_metastore_hosts: hive_conf_dirs_list.append(hive_server_conf_dir) -elif status_params.role == "HIVE_SERVER" and hive_server_hosts is not None and hostname in hive_server_host: +elif status_params.role == "HIVE_SERVER" and hive_server_hosts is not None and hostname in hive_server_hosts: hive_conf_dirs_list.append(hive_server_conf_dir) elif status_params.role == "HIVE_SERVER_INTERACTIVE" and hive_server_interactive_hosts is not None and hostname in hive_server_interactive_hosts: hive_conf_dirs_list.append(status_params.hive_server_interactive_conf_dir) http://git-wip-us.apache.org/repos/asf/ambari/blob/36cc11b3/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py index a12d388..91f10d8 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/2.1.0.3.0/package/scripts/params_linux.py @@ -399,7 +399,7 @@ hive_conf_dirs_list = [hive_client_conf_dir] ranger_hive_component = status_params.SERVER_ROLE_DIRECTORY_MAP['HIVE_SERVER'] if status_params.role == "HIVE_METASTORE" and hive_metastore_hosts is not None and hostname in hive_metastore_hosts: hive_conf_dirs_list.append(hive_server_conf_dir) -elif status_params.role == "HIVE_SERVER" and hive_server_hosts is not None and hostname in hive_server_host: +elif status_params.role == "HIVE_SERVER" and hive_server_hosts is not None and hostname in hive_server_hosts: hive_conf_dirs_list.append(hive_server_conf_dir) elif status_params.role == "HIVE_SERVER_INTERACTIVE" and hive_server_interactive_hosts is not None and hostname in hive_server_interactive_hosts: hive_conf_dirs_list.append(status_params.hive_server_interactive_conf_dir) http://git-wip-us.apache.org/repos/asf/ambari/blob/36cc11b3/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index ec19724..e10e4cd 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -1926,9 +1926,9 @@ public class AmbariManagementControllerTest { } catch (Exception e) { // Expected } - + clusters.addCluster(cluster1, new StackId("HDP-0.1")); - + try { set1.clear(); HostRequest rInvalid1 = @@ -6988,7 +6988,7 @@ public class AmbariManagementControllerTest { Assert.assertEquals(1, responsesWithParams.size()); StackVersionResponse resp = responsesWithParams.iterator().next(); assertNotNull(resp.getUpgradePacks()); - assertEquals(13, resp.getUpgradePacks().size()); + assertEquals(14, resp.getUpgradePacks().size()); assertTrue(resp.getUpgradePacks().contains("upgrade_test")); } http://git-wip-us.apache.org/repos/asf/ambari/blob/36cc11b3/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java index b2555cd..a4f5e9a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/UpgradeResourceProviderTest.java @@ -45,6 +45,7 @@ import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.actionmanager.Stage; +import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.audit.AuditLogger; import org.apache.ambari.server.configuration.Configuration; @@ -95,10 +96,12 @@ import org.apache.ambari.server.state.UpgradeContext; import org.apache.ambari.server.state.UpgradeHelper; import org.apache.ambari.server.state.UpgradeState; import org.apache.ambari.server.state.stack.upgrade.Direction; +import org.apache.ambari.server.state.stack.upgrade.StageWrapper; import org.apache.ambari.server.state.stack.upgrade.UpgradeType; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.utils.StageUtils; import org.apache.ambari.server.view.ViewRegistry; +import org.apache.commons.lang3.StringUtils; import org.easymock.EasyMock; import org.easymock.EasyMockSupport; import org.junit.After; @@ -1656,6 +1659,57 @@ public class UpgradeResourceProviderTest extends EasyMockSupport { } /** + * Tests that commands created for {@link StageWrapper.Type#RU_TASKS} set the + * service and component on the {@link ExecutionCommand}. + * <p/> + * Without this, commands of this type would not be able to determine which + * service/component repository they should use when the command is scheduled + * to run. + * + * @throws Exception + */ + @Test + public void testExecutionCommandServiceAndComponent() throws Exception { + Map<String, Object> requestProps = new HashMap<>(); + requestProps.put(UpgradeResourceProvider.UPGRADE_CLUSTER_NAME, "c1"); + requestProps.put(UpgradeResourceProvider.UPGRADE_REPO_VERSION_ID, String.valueOf(repoVersionEntity2200.getId())); + requestProps.put(UpgradeResourceProvider.UPGRADE_PACK, "upgrade_execute_task_test"); + requestProps.put(UpgradeResourceProvider.UPGRADE_SKIP_PREREQUISITE_CHECKS, "true"); + requestProps.put(UpgradeResourceProvider.UPGRADE_DIRECTION, Direction.UPGRADE.name()); + + ResourceProvider upgradeResourceProvider = createProvider(amc); + + Request request = PropertyHelper.getCreateRequest(Collections.singleton(requestProps), null); + RequestStatus status = upgradeResourceProvider.createResources(request); + + Set<Resource> createdResources = status.getAssociatedResources(); + assertEquals(1, createdResources.size()); + Resource res = createdResources.iterator().next(); + Long id = (Long) res.getPropertyValue("Upgrade/request_id"); + assertNotNull(id); + assertEquals(Long.valueOf(1), id); + + + ActionManager am = injector.getInstance(ActionManager.class); + List<HostRoleCommand> commands = am.getRequestTasks(id); + + boolean foundActionExecuteCommand = false; + for (HostRoleCommand command : commands) { + ExecutionCommand executionCommand = command.getExecutionCommandWrapper().getExecutionCommand(); + if (StringUtils.equals(UpgradeResourceProvider.EXECUTE_TASK_ROLE, + executionCommand.getRole())) { + foundActionExecuteCommand = true; + assertNotNull(executionCommand.getServiceName()); + assertNotNull(executionCommand.getComponentName()); + } + } + + assertTrue( + "There was no task found with the role of " + UpgradeResourceProvider.EXECUTE_TASK_ROLE, + foundActionExecuteCommand); + } + + /** * */ private class MockModule implements Module { http://git-wip-us.apache.org/repos/asf/ambari/blob/36cc11b3/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml new file mode 100644 index 0000000..3e61696 --- /dev/null +++ b/ambari-server/src/test/resources/stacks/HDP/2.1.1/upgrades/upgrade_execute_task_test.xml @@ -0,0 +1,42 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<upgrade xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="upgrade-pack.xsd"> + <target>2.2.*.*</target> + <target-stack>HDP-2.2.0</target-stack> + <type>ROLLING</type> + <prerequisite-checks/> + + <order> + <group xsi:type="cluster" name="CLUSTER_SERVER_ACTIONS" title="Cluster Server Actions"> + <execute-stage service="ZOOKEEPER" component="ZOOKEEPER_SERVER" title="Execute Foo/Bar Function"> + <task xsi:type="execute"> + <script>scripts/foo.py</script> + <function>bar</function> + </task> + </execute-stage> + </group> + </order> + + <processing> + <service name="ZOOKEEPER"> + <component name="ZOOKEEPER_SERVER"> + <upgrade /> + </component> + </service> + </processing> +</upgrade>