Repository: ambari Updated Branches: refs/heads/branch-2.5 53a39695a -> 4a7ed3c29
AMBARI-19423. Change HostStackVersionResourceProvider to be able to install packages on single host not belonging to any cluster (magyari_sandor) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4a7ed3c2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4a7ed3c2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4a7ed3c2 Branch: refs/heads/branch-2.5 Commit: 4a7ed3c29e6f32d6bfb53da0ab990bc0a5de87d0 Parents: 53a3969 Author: Sandor Magyari <smagy...@hortonworks.com> Authored: Wed Jan 4 13:44:25 2017 +0100 Committer: Sandor Magyari <smagy...@hortonworks.com> Committed: Wed Jan 18 12:02:09 2017 +0100 ---------------------------------------------------------------------- .../controller/AmbariActionExecutionHelper.java | 33 ++++-- .../HostStackVersionResourceProvider.java | 117 +++++++++++++++---- .../HostStackVersionResourceProviderTest.java | 96 +++++++++++++++ 3 files changed, 211 insertions(+), 35 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/4a7ed3c2/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index 4fa942f..ec0f7d0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -227,17 +227,28 @@ public class AmbariActionExecutionHelper { || targetHostType.equals(TargetHostType.MAJORITY); } - /** * Add tasks to the stage based on the requested action execution - * * @param actionContext the context associated with the action * @param stage stage into which tasks must be inserted * @param requestParams all request parameters (may be null) * @throws AmbariException if the task can not be added */ public void addExecutionCommandsToStage(final ActionExecutionContext actionContext, Stage stage, - Map<String, String> requestParams) + Map<String, String> requestParams) throws AmbariException { + addExecutionCommandsToStage(actionContext, stage, requestParams, true); + } + + /** + * Add tasks to the stage based on the requested action execution + * @param actionContext + * @param stage + * @param requestParams + * @param checkHostIsMemberOfCluster if true AmbariException will be thrown in case host is not member of cluster. + * @throws AmbariException + */ + public void addExecutionCommandsToStage(final ActionExecutionContext actionContext, Stage stage, + Map<String, String> requestParams, boolean checkHostIsMemberOfCluster) throws AmbariException { String actionName = actionContext.getActionName(); @@ -331,13 +342,15 @@ public class AmbariActionExecutionHelper { + "actionName=" + actionContext.getActionName()); } - // Compare specified hosts to available hosts - if (!resourceFilter.getHostNames().isEmpty() && !candidateHosts.isEmpty()) { - for (String hostname : resourceFilter.getHostNames()) { - if (!candidateHosts.contains(hostname)) { - throw new AmbariException("Request specifies host " + hostname + - " but it is not a valid host based on the " + - "target service=" + serviceName + " and component=" + componentName); + if (checkHostIsMemberOfCluster) { + // Compare specified hosts to available hosts + if (!resourceFilter.getHostNames().isEmpty() && !candidateHosts.isEmpty()) { + for (String hostname : resourceFilter.getHostNames()) { + if (!candidateHosts.contains(hostname)) { + throw new AmbariException("Request specifies host " + hostname + + " but it is not a valid host based on the " + + "target service=" + serviceName + " and component=" + componentName); + } } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/4a7ed3c2/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java index b8d7381..364a61e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java @@ -66,6 +66,7 @@ import org.apache.ambari.server.state.ServiceOsSpecific; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; @@ -91,6 +92,18 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource protected static final String HOST_STACK_VERSION_REPOSITORIES_PROPERTY_ID = PropertyHelper.getPropertyId("HostStackVersions", "repositories"); protected static final String HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID = PropertyHelper.getPropertyId("HostStackVersions", "repository_version"); + /** + * Whether to force creating of install command on a host which is not member of any cluster yet. + */ + protected static final String HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID = PropertyHelper + .getPropertyId("HostStackVersions", "force_non_member_install"); + + /** + * In case of force_non_member_install = true a list of component names must be provided in the request. + */ + protected static final String HOST_STACK_VERSION_COMPONENT_NAMES_PROPERTY_ID = PropertyHelper.getPropertyId("HostStackVersions", "components"); + protected static final String COMPONENT_NAME_PROPERTY_ID = "name"; + protected static final String INSTALL_PACKAGES_ACTION = "install_packages"; protected static final String INSTALL_PACKAGES_FULL_NAME = "Install version"; @@ -111,7 +124,9 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource HOST_STACK_VERSION_VERSION_PROPERTY_ID, HOST_STACK_VERSION_STATE_PROPERTY_ID, HOST_STACK_VERSION_REPOSITORIES_PROPERTY_ID, - HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID); + HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID, + HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID, + HOST_STACK_VERSION_COMPONENT_NAMES_PROPERTY_ID); private static Map<Type, String> keyPropertyIds = new HashMap<Type, String>() { { @@ -261,9 +276,36 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource String.format("The required property %s is not defined", requiredProperty)); } - String clName = (String) propertyMap.get(HOST_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID); + String clName = (String) propertyMap.get (HOST_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID); hostName = (String) propertyMap.get(HOST_STACK_VERSION_HOST_NAME_PROPERTY_ID); desiredRepoVersion = (String) propertyMap.get(HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID); + stackName = (String) propertyMap.get(HOST_STACK_VERSION_STACK_PROPERTY_ID); + stackVersion = (String) propertyMap.get(HOST_STACK_VERSION_VERSION_PROPERTY_ID); + + boolean forceInstallOnNonMemberHost = false; + Set<Map<String, String>> componentNames = null; + String forceInstallOnNonMemberHostString = (String) propertyMap.get + (HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID); + + if (BooleanUtils.toBoolean(forceInstallOnNonMemberHostString)) { + forceInstallOnNonMemberHost = true; + componentNames = (Set<Map<String, String>>) propertyMap.get(HOST_STACK_VERSION_COMPONENT_NAMES_PROPERTY_ID); + if (componentNames == null) { + throw new IllegalArgumentException("In case " + HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID + " is set to true, the list of " + + "components should be specified in request."); + } + } + + RequestStageContainer req = createInstallPackagesRequest(hostName, desiredRepoVersion, stackName, stackVersion, + clName, forceInstallOnNonMemberHost, componentNames); + return getRequestStatus(req.getRequestStatusResponse()); + } + + private RequestStageContainer createInstallPackagesRequest(String hostName, final String desiredRepoVersion, + String stackName, String stackVersion, String clName, + boolean forceInstallOnNonMemberHost, + Set<Map<String, String>> componentNames) + throws NoSuchParentResourceException, SystemException { Host host; try { @@ -275,8 +317,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource AmbariManagementController managementController = getManagementController(); AmbariMetaInfo ami = managementController.getAmbariMetaInfo(); - stackName = (String) propertyMap.get(HOST_STACK_VERSION_STACK_PROPERTY_ID); - stackVersion = (String) propertyMap.get(HOST_STACK_VERSION_VERSION_PROPERTY_ID); final StackId stackId = new StackId(stackName, stackVersion); if (!ami.isSupportedStack(stackName, stackVersion)) { throw new NoSuchParentResourceException(String.format("Stack %s is not supported", @@ -327,17 +367,19 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource HostVersionEntity hostVersEntity = hostVersionDAO.findByClusterStackVersionAndHost(clName, stackId, desiredRepoVersion, hostName); - if (hostVersEntity == null) { - throw new IllegalArgumentException(String.format( - "Repo version %s for stack %s is not available for host %s", - desiredRepoVersion, stackId, hostName)); - } - if (hostVersEntity.getState() != RepositoryVersionState.INSTALLED && - hostVersEntity.getState() != RepositoryVersionState.INSTALL_FAILED && - hostVersEntity.getState() != RepositoryVersionState.OUT_OF_SYNC) { - throw new UnsupportedOperationException(String.format("Repo version %s for stack %s " + - "for host %s is in %s state. Can not transition to INSTALLING state", - desiredRepoVersion, stackId, hostName, hostVersEntity.getState().toString())); + if (!forceInstallOnNonMemberHost) { + if (hostVersEntity == null) { + throw new IllegalArgumentException(String.format( + "Repo version %s for stack %s is not available for host %s", + desiredRepoVersion, stackId, hostName)); + } + if (hostVersEntity.getState() != RepositoryVersionState.INSTALLED && + hostVersEntity.getState() != RepositoryVersionState.INSTALL_FAILED && + hostVersEntity.getState() != RepositoryVersionState.OUT_OF_SYNC) { + throw new UnsupportedOperationException(String.format("Repo version %s for stack %s " + + "for host %s is in %s state. Can not transition to INSTALLING state", + desiredRepoVersion, stackId, hostName, hostVersEntity.getState().toString())); + } } List<OperatingSystemEntity> operatingSystems = repoVersionEnt.getOperatingSystems(); @@ -357,9 +399,34 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource // For every host at cluster, determine packages for all installed services List<ServiceOsSpecific.Package> packages = new ArrayList<>(); Set<String> servicesOnHost = new HashSet<>(); - List<ServiceComponentHost> components = cluster.getServiceComponentHosts(host.getHostName()); - for (ServiceComponentHost component : components) { - servicesOnHost.add(component.getServiceName()); + + if (forceInstallOnNonMemberHost) { + for (Map<String, String> componentProperties : componentNames) { + + String componentName = componentProperties.get(COMPONENT_NAME_PROPERTY_ID); + if (StringUtils.isEmpty(componentName)) { + throw new IllegalArgumentException("Components list contains a component with no 'name' property"); + } + + String serviceName = null; + try { + serviceName = ami.getComponentToService(stackName, stackVersion, componentName.trim().toUpperCase()); + if (serviceName == null) { + throw new IllegalArgumentException("Service not found for component : " + componentName); + } + servicesOnHost.add(serviceName); + } catch (AmbariException e) { + LOG.error("Service not found for component {}!", componentName, e); + throw new IllegalArgumentException("Service not found for component : " + componentName); + } + + } + + } else { + List<ServiceComponentHost> components = cluster.getServiceComponentHosts(host.getHostName()); + for (ServiceComponentHost component : components) { + servicesOnHost.add(component.getServiceName()); + } } List<String> blacklistedPackagePrefixes = configuration.getRollingUpgradeSkipPackagesPrefixes(); for (String serviceName : servicesOnHost) { @@ -409,7 +476,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource params.put(KeyNames.PACKAGE_VERSION, xml.getPackageVersion(osFamily)); } - // Create custom action RequestResourceFilter filter = new RequestResourceFilter(null, null, Collections.singletonList(hostName)); @@ -452,21 +518,22 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource req.addStages(Collections.singletonList(stage)); try { - actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, null); + actionExecutionHelper.get().addExecutionCommandsToStage(actionContext, stage, null, !forceInstallOnNonMemberHost); } catch (AmbariException e) { throw new SystemException("Can not modify stage", e); } try { - hostVersEntity.setState(RepositoryVersionState.INSTALLING); - hostVersionDAO.merge(hostVersEntity); - - cluster.recalculateClusterVersionState(repoVersionEnt); + if (!forceInstallOnNonMemberHost) { + hostVersEntity.setState(RepositoryVersionState.INSTALLING); + hostVersionDAO.merge(hostVersEntity); + cluster.recalculateClusterVersionState(repoVersionEnt); + } req.persist(); } catch (AmbariException e) { throw new SystemException("Can not persist request", e); } - return getRequestStatus(req.getRequestStatusResponse()); + return req; } http://git-wip-us.apache.org/repos/asf/ambari/blob/4a7ed3c2/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java index 7fc8a90..ecbd273 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java @@ -32,6 +32,7 @@ import static org.easymock.EasyMock.verify; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -291,6 +292,101 @@ public class HostStackVersionResourceProviderTest { } @Test + public void testCreateResources_on_host_not_belonging_To_any_cluster() throws Exception { + StackId stackId = new StackId("HDP", "2.0.1"); + + final Host host1 = createNiceMock("host1", Host.class); + expect(host1.getHostName()).andReturn("host1").anyTimes(); + expect(host1.getOsFamily()).andReturn("redhat6").anyTimes(); + replay(host1); + Map<String, Host> hostsForCluster = new HashMap<>(); + + ServiceComponentHost sch = createMock(ServiceComponentHost.class); + + final ServiceOsSpecific.Package hivePackage = new ServiceOsSpecific.Package(); + hivePackage.setName("hive"); + final ServiceOsSpecific.Package mysqlPackage = new ServiceOsSpecific.Package(); + mysqlPackage.setName("mysql"); + mysqlPackage.setSkipUpgrade(Boolean.TRUE); + List<ServiceOsSpecific.Package> packages = Arrays.asList(hivePackage, mysqlPackage); + + AbstractControllerResourceProvider.init(resourceProviderFactory); + + Map<String, Map<String, String>> hostConfigTags = new HashMap<>(); + expect(configHelper.getEffectiveDesiredTags(anyObject(ClusterImpl.class), anyObject(String.class))).andReturn(hostConfigTags); + + expect(managementController.getClusters()).andReturn(clusters).anyTimes(); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(managementController.getActionManager()).andReturn(actionManager).anyTimes(); + expect(managementController.getJdkResourceUrl()).andReturn("/JdkResourceUrl").anyTimes(); + expect(managementController.getPackagesForServiceHost(anyObject(ServiceInfo.class), + anyObject(Map.class), anyObject(String.class))).andReturn(packages).anyTimes(); + + expect(resourceProviderFactory.getHostResourceProvider(anyObject(Set.class), anyObject(Map.class), + eq(managementController))).andReturn(csvResourceProvider).anyTimes(); + + expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster); + expect(clusters.getHost(anyObject(String.class))).andReturn(host1); + expect(cluster.getHosts()).andReturn(hostsForCluster.values()).atLeastOnce(); + expect(cluster.getServices()).andReturn(new HashMap<String, Service>()).anyTimes(); + expect(cluster.getCurrentStackVersion()).andReturn(stackId); + + expect( + repositoryVersionDAOMock.findByStackAndVersion( + anyObject(StackId.class), + anyObject(String.class))).andReturn(repoVersion); + + expect(actionManager.getRequestTasks(anyLong())).andReturn(Collections.<HostRoleCommand>emptyList()).anyTimes(); + + StageUtils.setTopologyManager(injector.getInstance(TopologyManager.class)); + StageUtils.setConfiguration(injector.getInstance(Configuration.class)); + + // replay + replay(managementController, response, clusters, resourceProviderFactory, csvResourceProvider, + cluster, repositoryVersionDAOMock, configHelper, sch, actionManager, hostVersionEntityMock, hostVersionDAOMock); + + ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( + type, + PropertyHelper.getPropertyIds(type), + PropertyHelper.getKeyPropertyIds(type), + managementController); + + injector.injectMembers(provider); + + // add the property map to a set for the request. add more maps for multiple creates + Set<Map<String, Object>> propertySet = new LinkedHashSet<>(); + + Map<String, Object> properties = new LinkedHashMap<>(); + + // add properties to the request map + properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID, "Cluster100"); + properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_REPO_VERSION_PROPERTY_ID, "2.2.0.1-885"); + properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_STACK_PROPERTY_ID, "HDP"); + properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_VERSION_PROPERTY_ID, "2.0.1"); + properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_HOST_NAME_PROPERTY_ID, "host1"); + Set<Map<String, String>> components = new HashSet<>(); + Map<String, String> hiveMetastoreComponent = new HashMap<>(); + hiveMetastoreComponent.put("name", "HIVE_METASTORE"); + components.add(hiveMetastoreComponent); + Map<String, String> hiveServerstoreComponent = new HashMap<>(); + hiveServerstoreComponent.put("name", "HIVE_SERVER"); + components.add(hiveServerstoreComponent); + properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_COMPONENT_NAMES_PROPERTY_ID, components); + properties.put(HostStackVersionResourceProvider.HOST_STACK_VERSION_FORCE_INSTALL_ON_NON_MEMBER_HOST_PROPERTY_ID, + "true"); + + propertySet.add(properties); + + // create the request + Request request = PropertyHelper.getCreateRequest(propertySet, null); + + provider.createResources(request); + + // verify + verify(managementController, response, clusters); + } + + @Test public void testCreateResources_in_out_of_sync_state() throws Exception { StackId stackId = new StackId("HDP", "2.0.1");