AMBARI-20774 - Service Upgrade VDF Creates Host Version Entries For All Hosts With INSTALLING (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/56f838b5 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/56f838b5 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/56f838b5 Branch: refs/heads/trunk Commit: 56f838b5b0a5681dffb703f503fcf443cb3b6dbd Parents: b013be0 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Mon Apr 17 17:19:45 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Tue Apr 18 14:13:21 2017 -0400 ---------------------------------------------------------------------- .../ClusterStackVersionResourceProvider.java | 136 ++++----- .../org/apache/ambari/server/state/Cluster.java | 40 ++- .../server/state/cluster/ClusterImpl.java | 169 +++++++----- .../server/upgrade/UpgradeCatalog300.java | 8 +- .../ServicesNamenodeTruncateCheckTest.java | 2 +- .../AmbariManagementControllerImplTest.java | 6 +- ...ClusterStackVersionResourceProviderTest.java | 273 +++++++++++-------- .../HostVersionOutOfSyncListenerTest.java | 42 ++- .../server/state/cluster/ClusterTest.java | 62 +++-- 9 files changed, 413 insertions(+), 325 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index 9c54a9d..e39588e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -20,7 +20,6 @@ package org.apache.ambari.server.controller.internal; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; @@ -77,7 +76,6 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.Host; -import org.apache.ambari.server.state.MaintenanceState; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.RepositoryVersionState; import org.apache.ambari.server.state.ServiceComponentHost; @@ -89,6 +87,7 @@ import org.apache.ambari.server.utils.StageUtils; import org.apache.ambari.server.utils.VersionUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.math.NumberUtils; +import org.apache.hadoop.metrics2.sink.relocated.google.common.collect.Lists; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; @@ -350,10 +349,10 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou stackId = currentStackVersion; } - RepositoryVersionEntity repoVersionEnt = repositoryVersionDAO.findByStackAndVersion( + RepositoryVersionEntity repoVersionEntity = repositoryVersionDAO.findByStackAndVersion( stackId, desiredRepoVersion); - if (repoVersionEnt == null) { + if (repoVersionEntity == null) { throw new IllegalArgumentException(String.format( "Repo version %s is not available for stack %s", desiredRepoVersion, stackId)); @@ -361,14 +360,40 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou VersionDefinitionXml desiredVersionDefinition = null; try { - desiredVersionDefinition = repoVersionEnt.getRepositoryXml(); + desiredVersionDefinition = repoVersionEntity.getRepositoryXml(); } catch (Exception e) { throw new IllegalArgumentException( String.format("Version %s is backed by a version definition, but it could not be parsed", desiredRepoVersion), e); } - // get all of the host eligible for stack distribution - List<Host> hosts = getHostsForStackDistribution(cluster); + // if true, then we need to force all new host versions into the INSTALLED state + boolean forceInstalled = Boolean.parseBoolean((String)propertyMap.get( + CLUSTER_STACK_VERSION_FORCE)); + + try { + // either create the necessary host version entries, or set them to INSTALLING when attempting to re-distribute an existing version + return createOrUpdateHostVersions(cluster, repoVersionEntity, desiredVersionDefinition, + stackId, forceInstalled, propertyMap); + } catch (AmbariException e) { + throw new SystemException("Can not persist request", e); + } + } + + @Transactional + RequestStatus createOrUpdateHostVersions(Cluster cluster, + RepositoryVersionEntity repoVersionEntity, VersionDefinitionXml versionDefinitionXml, + StackId stackId, boolean forceInstalled, Map<String, Object> propertyMap) + throws AmbariException, SystemException { + + final String clusterName = cluster.getClusterName(); + final String authName = getManagementController().getAuthName(); + final String desiredRepoVersion = repoVersionEntity.getVersion(); + + ClusterVersionEntity clusterVersionEntity = clusterVersionDAO.findByClusterAndStackAndVersion( + clusterName, stackId, desiredRepoVersion); + + // get all of the hosts eligible for stack distribution + List<Host> hosts = Lists.newArrayList(cluster.getHosts()); /* If there is a repository that is already ATTEMPTED to be installed and the version @@ -384,7 +409,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou install by name. if the package-version is not known, then the 'newest' is ALWAYS installed. In this case, 2.5.0.0. 2.4 is never picked up. */ - for (ClusterVersionEntity clusterVersion : clusterVersionDAO.findByCluster(clName)) { + for (ClusterVersionEntity clusterVersion : clusterVersionDAO.findByCluster(cluster.getClusterName())) { RepositoryVersionEntity clusterRepoVersion = clusterVersion.getRepositoryVersion(); int compare = compareVersions(clusterRepoVersion.getVersion(), desiredRepoVersion); @@ -397,18 +422,18 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou // !!! the version is greater to the one to install // if the stacks are different, then don't fail (further check same-stack version strings) - if (!StringUtils.equals(clusterRepoVersion.getStackName(), repoVersionEnt.getStackName())) { + if (!StringUtils.equals(clusterRepoVersion.getStackName(), repoVersionEntity.getStackName())) { continue; } // if there is no backing VDF for the desired version, allow the operation (legacy behavior) - if (null == desiredVersionDefinition) { + if (null == versionDefinitionXml) { continue; } // backing VDF does not define the package version for any of the hosts, cannot install (allows a VDF with package-version) for (Host host : hosts) { - if (StringUtils.isBlank(desiredVersionDefinition.getPackageVersion(host.getOsFamily()))) { + if (StringUtils.isBlank(versionDefinitionXml.getPackageVersion(host.getOsFamily()))) { String msg = String.format("Ambari cannot install version %s. Version %s is already installed.", desiredRepoVersion, clusterRepoVersion.getVersion()); throw new IllegalArgumentException(msg); @@ -416,49 +441,17 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou } } - // if true, then we need to force all new host versions into the INSTALLED state - boolean forceInstalled = Boolean.parseBoolean((String)propertyMap.get( - CLUSTER_STACK_VERSION_FORCE)); - - final RequestStatusResponse response; - - try { - if (forceInstalled) { - createHostVersions(cluster, hosts, stackId, desiredRepoVersion, RepositoryVersionState.INSTALLED); - response = null; - } else { - createHostVersions(cluster, hosts, stackId, desiredRepoVersion, - RepositoryVersionState.INSTALLING); - - RequestStageContainer installRequest = createOrchestration(cluster, stackId, hosts, - repoVersionEnt, desiredVersionDefinition, propertyMap); - - response = installRequest.getRequestStatusResponse(); - } - } catch (AmbariException e) { - throw new SystemException("Can not persist request", e); + RepositoryVersionState repositoryVersionState = RepositoryVersionState.INSTALLING; + if (forceInstalled) { + repositoryVersionState = RepositoryVersionState.INSTALLED; } - return getRequestStatus(response); - } - - @Transactional - void createHostVersions(Cluster cluster, List<Host> hosts, StackId stackId, - String desiredRepoVersion, RepositoryVersionState repoState) - throws AmbariException, SystemException { - final String clusterName = cluster.getClusterName(); - final String authName = getManagementController().getAuthName(); - - ClusterVersionEntity clusterVersionEntity = clusterVersionDAO.findByClusterAndStackAndVersion( - clusterName, stackId, desiredRepoVersion); - + // if there is no cluster version entity, then create one if (clusterVersionEntity == null) { try { // Create/persist new cluster stack version - cluster.createClusterVersion(stackId, desiredRepoVersion, authName, repoState); - - clusterVersionEntity = clusterVersionDAO.findByClusterAndStackAndVersion(clusterName, - stackId, desiredRepoVersion); + clusterVersionEntity = cluster.createClusterVersion(stackId, desiredRepoVersion, authName, + repositoryVersionState); } catch (AmbariException e) { throw new SystemException( String.format("Can not create cluster stack version %s for cluster %s", @@ -466,20 +459,22 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou } } else { // Move cluster version into the specified state (retry installation) - cluster.transitionClusterVersion(stackId, desiredRepoVersion, repoState); + cluster.transitionClusterVersion(stackId, desiredRepoVersion, repositoryVersionState); } - // Will also initialize all Host Versions to the specified state state. - cluster.transitionHosts(clusterVersionEntity, repoState); + // the cluster will create/update all of the host versions to the correct state + List<Host> hostsNeedingInstallCommands = cluster.transitionHostsToInstalling( + clusterVersionEntity, repoVersionEntity, versionDefinitionXml, forceInstalled); - // Directly transition host versions to NOT_REQUIRED for hosts that don't - // have versionable components - for (Host host : hosts) { - if (!host.hasComponentsAdvertisingVersions(stackId)) { - transitionHostVersionToNotRequired(host, cluster, - clusterVersionEntity.getRepositoryVersion()); - } + RequestStatusResponse response = null; + if (!forceInstalled) { + RequestStageContainer installRequest = createOrchestration(cluster, stackId, + hostsNeedingInstallCommands, repoVersionEntity, versionDefinitionXml, propertyMap); + + response = installRequest.getRequestStatusResponse(); } + + return getRequestStatus(response); } @Transactional @@ -910,29 +905,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou } /** - * Gets all of the hosts in a cluster which are not in "maintenance mode" and - * are considered to be healthy. In the case of stack distribution, a host - * must be explicitely marked as being in maintenance mode for it to be - * considered as unhealthy. - * - * @param cluster - * the cluster (not {@code null}). - * @return the list of hosts that are not in maintenance mode and are - * elidgable to have a stack distributed to them. - */ - private List<Host> getHostsForStackDistribution(Cluster cluster) { - Collection<Host> hosts = cluster.getHosts(); - List<Host> healthyHosts = new ArrayList<>(hosts.size()); - for (Host host : hosts) { - if (host.getMaintenanceState(cluster.getClusterId()) == MaintenanceState.OFF) { - healthyHosts.add(host); - } - } - - return healthyHosts; - } - - /** * Updates the version states. Transactional to ensure only one transaction for all updates * @param clusterId the cluster * @param current the repository that is current for the cluster http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java index c961995..25b4a19 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/Cluster.java @@ -36,6 +36,7 @@ import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.orm.entities.UpgradeEntity; import org.apache.ambari.server.security.authorization.AuthorizationException; import org.apache.ambari.server.state.configgroup.ConfigGroup; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.scheduler.RequestExecution; import com.google.common.collect.ListMultimap; @@ -229,25 +230,40 @@ public interface Cluster { /** * Creates or updates host versions for all of the hosts within a cluster * based on state of cluster stack version. This is used to transition all - * hosts into the specified state. + * hosts into the correct state (which may not be + * {@link RepositoryVersionState#INSTALLING}). * <p/> * The difference between this method compared to * {@link Cluster#mapHostVersions} is that it affects all hosts (not only * missing hosts). * <p/> * Hosts that are in maintenance mode will be transititioned directly into - * {@link RepositoryVersionState#OUT_OF_SYNC} instead. + * {@link RepositoryVersionState#OUT_OF_SYNC} instead. Hosts which do not need + * the version distributed to them will move into the + * {@link RepositoryVersionState#NOT_REQUIRED} state. * * @param sourceClusterVersion * cluster version to be queried for a stack name/version info and * desired RepositoryVersionState. The only valid state of a cluster * version is {@link RepositoryVersionState#INSTALLING} - * @param state - * the state to transition the cluster's hosts to. + * @param repoVersionEntity + * the repository that the hosts are being transitioned for (not + * {@code null}). + * @param versionDefinitionXml + * the VDF, or {@code null} if none. + * @param forceInstalled + * if {@code true}, then this will transition everything directly to + * {@link RepositoryVersionState#INSTALLED} instead of + * {@link RepositoryVersionState#INSTALLING}. Hosts which should + * received other states (like + * {@link RepositoryVersionState#NOT_REQUIRED} will continue to + * receive those states. + * @return a list of hosts which need the repository installed. * @throws AmbariException */ - void transitionHosts(ClusterVersionEntity sourceClusterVersion, RepositoryVersionState state) - throws AmbariException; + List<Host> transitionHostsToInstalling(ClusterVersionEntity sourceClusterVersion, + RepositoryVersionEntity repoVersionEntity, VersionDefinitionXml versionDefinitionXml, + boolean forceInstalled) throws AmbariException; /** * For a given host, will either either update an existing Host Version Entity for the given version, or create @@ -280,8 +296,9 @@ public interface Cluster { * Create a cluster version for the given stack and version, whose initial * state must either be either {@link RepositoryVersionState#UPGRADING} (if no * other cluster version exists) or {@link RepositoryVersionState#INSTALLING} - * (if at exactly one CURRENT cluster version already exists) or {@link RepositoryVersionState#INIT} - * (if the cluster is being created using a specific repository version). + * (if at exactly one CURRENT cluster version already exists) or + * {@link RepositoryVersionState#INIT} (if the cluster is being created using + * a specific repository version). * * @param stackId * Stack ID @@ -291,9 +308,10 @@ public interface Cluster { * User performing the operation * @param state * Initial state + * @return the newly created and persisted {@link ClusterVersionEntity}. * @throws AmbariException */ - void createClusterVersion(StackId stackId, String version, + ClusterVersionEntity createClusterVersion(StackId stackId, String version, String userName, RepositoryVersionState state) throws AmbariException; /** @@ -675,10 +693,10 @@ public interface Cluster { * Gets an {@link UpgradeEntity} if there is an upgrade in progress or an * upgrade that has been suspended. This will return the associated * {@link UpgradeEntity} if it exists. - * + * * @return an upgrade which will either be in progress or suspended, or * {@code null} if none. - * + * */ UpgradeEntity getUpgradeInProgress(); http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java index b7cc4cd..2e89bb8 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/cluster/ClusterImpl.java @@ -124,6 +124,7 @@ import org.apache.ambari.server.state.HostHealthStatus; import org.apache.ambari.server.state.HostState; import org.apache.ambari.server.state.MaintenanceState; import org.apache.ambari.server.state.PropertyInfo; +import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.RepositoryVersionState; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.Service; @@ -141,6 +142,7 @@ import org.apache.ambari.server.state.UpgradeContextFactory; import org.apache.ambari.server.state.configgroup.ConfigGroup; import org.apache.ambari.server.state.configgroup.ConfigGroupFactory; import org.apache.ambari.server.state.fsm.InvalidStateTransitionException; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.scheduler.RequestExecution; import org.apache.ambari.server.state.scheduler.RequestExecutionFactory; import org.apache.ambari.server.state.stack.upgrade.Direction; @@ -1172,99 +1174,115 @@ public class ClusterImpl implements Cluster { * {@inheritDoc} */ @Override - public void transitionHosts(ClusterVersionEntity sourceClusterVersion, - RepositoryVersionState state) throws AmbariException { + @Transactional + public List<Host> transitionHostsToInstalling(ClusterVersionEntity sourceClusterVersion, + RepositoryVersionEntity repoVersionEntity, VersionDefinitionXml versionDefinitionXml, + boolean forceInstalled) throws AmbariException { if (sourceClusterVersion == null) { throw new AmbariException("Could not find current stack version of cluster " + getClusterName()); } - if (state != sourceClusterVersion.getState()) { - throw new AmbariException("Unable to transition cluster hosts into " + state + if (RepositoryVersionState.INSTALLING != sourceClusterVersion.getState()) { + throw new AmbariException( + "Unable to transition cluster hosts into " + RepositoryVersionState.INSTALLING + ". The only valid state is " + sourceClusterVersion.getState()); } - Map<String, Host> hosts = clusters.getHostsForCluster(getClusterName()); - Set<String> existingHostsWithClusterStackAndVersion = new HashSet<>(); - HashMap<String, HostVersionEntity> existingHostStackVersions = new HashMap<>(); + // the hosts to return so that INSTALL commands can be generated for them + final List<Host> hostsRequiringInstallation; clusterGlobalLock.writeLock().lock(); try { - StackEntity repoVersionStackEntity = sourceClusterVersion.getRepositoryVersion().getStack(); - StackId repoVersionStackId = new StackId(repoVersionStackEntity); - List<HostVersionEntity> existingHostVersionEntities = hostVersionDAO.findByClusterStackAndVersion( - getClusterName(), repoVersionStackId, - sourceClusterVersion.getRepositoryVersion().getVersion()); + // get this once for easy lookup later + Map<String, Host> hosts = clusters.getHostsForCluster(getClusterName()); + hostsRequiringInstallation = new ArrayList<>(hosts.size()); - // for each host that already has a stack and version, keep track of them - for (HostVersionEntity entity : existingHostVersionEntities) { - String hostName = entity.getHostName(); - existingHostsWithClusterStackAndVersion.add(hostName); - existingHostStackVersions.put(hostName, entity); - } + // for every host, either create or update the host version to the right + // state - starting with STATE + Collection<HostEntity> hostEntities = getClusterEntity().getHostEntities(); - // find any hosts that do not have the stack/repo version already - Sets.SetView<String> hostsMissingRepoVersion = Sets.difference( - hosts.keySet(), existingHostsWithClusterStackAndVersion); + for (HostEntity hostEntity : hostEntities) { + // start with INSTALLING + RepositoryVersionState state = RepositoryVersionState.INSTALLING; + if (forceInstalled) { + state = RepositoryVersionState.INSTALLED; + } - createOrUpdateHostVersionToState(sourceClusterVersion, hosts, - existingHostStackVersions, hostsMissingRepoVersion, state); - } finally { - clusterGlobalLock.writeLock().unlock(); - } - } + // is this host version not required b/c of versionable components + Host host = hosts.get(hostEntity.getHostName()); + if (!host.hasComponentsAdvertisingVersions(desiredStackVersion)) { + state = RepositoryVersionState.NOT_REQUIRED; + } - /** - * Moved out to a separate method due to performance reasons - * Iterates over all hosts and creates or transitions existing host versions - * to a given state. If host version for desired stack/version does not exist, - * host version is created and initialized to a given state. Otherwise, existing - * host version state is updated - * Hosts in maintenance mode are auto skipped. - * - * @param sourceClusterVersion cluster version to be queried for a stack - * name/version info when creating a new host version - * @param hosts list of all hosts - * @param existingHostStackVersions map of existing host versions to be updated - * @param hostsMissingRepoVersion set of hostnames of hosts that have no desired host version - * @param newState target host version state for transition - */ - @Transactional - void createOrUpdateHostVersionToState(ClusterVersionEntity sourceClusterVersion, - Map<String, Host> hosts, HashMap<String, HostVersionEntity> existingHostStackVersions, - Sets.SetView<String> hostsMissingRepoVersion, RepositoryVersionState newState) { + // if the repository is still required, check against the repo type + if (state != RepositoryVersionState.NOT_REQUIRED) { + if (repoVersionEntity.getType() != RepositoryType.STANDARD) { + // does the host gets a different repo state based on VDF and repo + // type + boolean hostRequiresRepository = false; + Set<String> servicesInRepository = versionDefinitionXml.getAvailableServiceNames(); + + List<ServiceComponentHost> schs = getServiceComponentHosts(hostEntity.getHostName()); + for (ServiceComponentHost serviceComponentHost : schs) { + String serviceName = serviceComponentHost.getServiceName(); + if (servicesInRepository.contains(serviceName)) { + hostRequiresRepository = true; + break; + } + } - for (String hostname : hosts.keySet()) { - // start off with the requested new state for each host - RepositoryVersionState repositoryVersionState = newState; + // if not required, then move onto the next host + if (!hostRequiresRepository) { + state = RepositoryVersionState.NOT_REQUIRED; + } + } + } - // if the host is in maintenance mode, that's an explicit marker which - // indicates that it should not be transitioned to INSTALLING; instead - // they will be transitioned to OUT_OF_SYNC - Host host = hosts.get(hostname); - if (host.getMaintenanceState(getClusterId()) != MaintenanceState.OFF) { - repositoryVersionState = RepositoryVersionState.OUT_OF_SYNC; - } + // last check if it's still required - check for MM + if (state != RepositoryVersionState.NOT_REQUIRED) { + if (host.getMaintenanceState(clusterId) != MaintenanceState.OFF) { + state = RepositoryVersionState.OUT_OF_SYNC; + } + } - if (hostsMissingRepoVersion.contains(hostname)) { - // Create new host stack version - HostEntity hostEntity = hostDAO.findByName(hostname); - HostVersionEntity hostVersionEntity = new HostVersionEntity(hostEntity, - sourceClusterVersion.getRepositoryVersion(), repositoryVersionState); + // now that the correct state is determdined for the host version, + // either update or create it + HostVersionEntity hostVersionEntity = null; + Collection<HostVersionEntity> hostVersions = hostEntity.getHostVersionEntities(); + for (HostVersionEntity existingHostVersion : hostVersions) { + if (existingHostVersion.getRepositoryVersion().getId() == repoVersionEntity.getId()) { + hostVersionEntity = existingHostVersion; + break; + } + } - LOG.info("Creating host version for {}, state={}, repo={} (repo_id={})", + if (null == hostVersionEntity) { + hostVersionEntity = new HostVersionEntity(hostEntity, repoVersionEntity, state); + hostVersionDAO.create(hostVersionEntity); + + // bi-directional association update + hostVersions.add(hostVersionEntity); + hostDAO.merge(hostEntity); + } else { + hostVersionEntity.setState(state); + hostVersionEntity = hostVersionDAO.merge(hostVersionEntity); + } + + LOG.info("Created host version for {}, state={}, repository version={} (repo_id={})", hostVersionEntity.getHostName(), hostVersionEntity.getState(), - hostVersionEntity.getRepositoryVersion().getVersion(), hostVersionEntity.getRepositoryVersion().getId()); + repoVersionEntity.getVersion(), repoVersionEntity.getId()); - hostVersionDAO.create(hostVersionEntity); - } else { - // Update existing host stack version - HostVersionEntity hostVersionEntity = existingHostStackVersions.get(hostname); - hostVersionEntity.setState(repositoryVersionState); - hostVersionEntity = hostVersionDAO.merge(hostVersionEntity); + if (state == RepositoryVersionState.INSTALLING) { + hostsRequiringInstallation.add(host); + } } + } finally { + clusterGlobalLock.writeLock().unlock(); } + + return hostsRequiringInstallation; } /** @@ -1597,11 +1615,11 @@ public class ClusterImpl implements Cluster { } @Override - public void createClusterVersion(StackId stackId, String version, + public ClusterVersionEntity createClusterVersion(StackId stackId, String version, String userName, RepositoryVersionState state) throws AmbariException { clusterGlobalLock.writeLock().lock(); try { - createClusterVersionInternal(stackId, version, userName, state); + return createClusterVersionInternal(stackId, version, userName, state); } finally { clusterGlobalLock.writeLock().unlock(); } @@ -1612,7 +1630,7 @@ public class ClusterImpl implements Cluster { * * This method is intended to be called only when cluster lock is already acquired. */ - private void createClusterVersionInternal(StackId stackId, String version, + private ClusterVersionEntity createClusterVersionInternal(StackId stackId, String version, String userName, RepositoryVersionState state) throws AmbariException { if (!ALLOWED_REPOSITORY_STATES.contains(state)) { throw new AmbariException("The allowed state for a new cluster version must be within " + ALLOWED_REPOSITORY_STATES); @@ -1630,9 +1648,8 @@ public class ClusterImpl implements Cluster { RepositoryVersionEntity repositoryVersionEntity = repositoryVersionDAO.findByStackAndVersion( stackId, version); if (repositoryVersionEntity == null) { - LOG.warn("Could not find repository version for stack=" + stackId - + ", version=" + version); - return; + throw new AmbariException( + "Unable to find repository version for stack " + stackId + " and version " + version); } ClusterEntity clusterEntity = getClusterEntity(); @@ -1642,6 +1659,8 @@ public class ClusterImpl implements Cluster { clusterVersionDAO.create(clusterVersionEntity); clusterEntity.getClusterVersionEntities().add(clusterVersionEntity); clusterEntity = clusterDAO.merge(clusterEntity); + + return clusterVersionEntity; } /** http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java index 0b1d538..4a3d0e0 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/upgrade/UpgradeCatalog300.java @@ -36,9 +36,9 @@ import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; import org.apache.ambari.server.controller.AmbariManagementController; -import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; import org.apache.ambari.server.controller.internal.CalculatedStatus; import org.apache.ambari.server.orm.DBAccessor; +import org.apache.ambari.server.orm.DBAccessor.DBColumnInfo; import org.apache.ambari.server.orm.dao.DaoUtils; import org.apache.ambari.server.orm.dao.RequestDAO; import org.apache.ambari.server.orm.entities.RequestEntity; @@ -291,7 +291,7 @@ public class UpgradeCatalog300 extends AbstractUpgradeCatalog { // the cluster configuration mapping table dbAccessor.dropTable(CLUSTER_CONFIG_MAPPING_TABLE); } - + /** * Updates Log Search configs. * @@ -311,12 +311,12 @@ public class UpgradeCatalog300 extends AbstractUpgradeCatalog { if (!configType.endsWith("-logsearch-conf")) { continue; } - + Set<String> removeProperties = new HashSet<>(); removeProperties.add("service_name"); removeProperties.add("component_mappings"); removeProperties.add("content"); - + removeConfigurationPropertiesFromCluster(cluster, configType, removeProperties); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java b/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java index 2954f0d..ca71e3f 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/checks/ServicesNamenodeTruncateCheckTest.java @@ -53,8 +53,8 @@ public class ServicesNamenodeTruncateCheckTest { private Clusters m_clusters = EasyMock.createMock(Clusters.class); private ServicesNamenodeTruncateCheck m_check = new ServicesNamenodeTruncateCheck(); - private RepositoryVersionDAO m_repositoryVersionDAO = EasyMock.createMock(RepositoryVersionDAO.class); private final Map<String, String> m_configMap = new HashMap<>(); + private RepositoryVersionDAO m_repositoryVersionDAO = EasyMock.createMock(RepositoryVersionDAO.class); @Before public void setup() throws Exception { http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java index 625ac8a..f35122a 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerImplTest.java @@ -2359,8 +2359,10 @@ public class AmbariManagementControllerImplTest { expect(cluster.getDesiredStackVersion()).andReturn(new StackId("HDP-2.1")).atLeastOnce(); // this getting called one time means the cluster version is getting created - cluster.createClusterVersion(anyObject(StackId.class), anyObject(String.class), anyObject(String.class), anyObject(RepositoryVersionState.class)); - expectLastCall().once(); + ClusterVersionEntity clusterVersionEntity = createNiceMock(ClusterVersionEntity.class); + expect(cluster.createClusterVersion(anyObject(StackId.class), anyObject(String.class), + anyObject(String.class), anyObject(RepositoryVersionState.class))).andReturn( + clusterVersionEntity).once(); expect(clusters.getCluster("c1")).andReturn(cluster).atLeastOnce(); http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java index 5ef31b5..8f7b31d 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java @@ -19,111 +19,112 @@ package org.apache.ambari.server.controller.internal; import static org.easymock.EasyMock.anyLong; - import static org.easymock.EasyMock.anyObject; - import static org.easymock.EasyMock.capture; - import static org.easymock.EasyMock.createMock; - import static org.easymock.EasyMock.createNiceMock; - import static org.easymock.EasyMock.eq; - import static org.easymock.EasyMock.expect; - import static org.easymock.EasyMock.expectLastCall; - import static org.easymock.EasyMock.replay; - import static org.easymock.EasyMock.verify; - - import java.io.File; - import java.io.FileInputStream; - import java.lang.reflect.Field; - import java.sql.SQLException; - import java.util.ArrayList; - import java.util.Arrays; - import java.util.Collections; - import java.util.HashMap; - import java.util.LinkedHashMap; - import java.util.LinkedHashSet; - import java.util.List; - import java.util.Map; - import java.util.Properties; - import java.util.Set; - - import org.apache.ambari.annotations.Experimental; - import org.apache.ambari.annotations.ExperimentalFeature; - import org.apache.ambari.server.AmbariException; - import org.apache.ambari.server.H2DatabaseCleaner; - import org.apache.ambari.server.Role; - import org.apache.ambari.server.actionmanager.ActionManager; - import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; - import org.apache.ambari.server.actionmanager.HostRoleCommand; - import org.apache.ambari.server.actionmanager.Stage; - import org.apache.ambari.server.actionmanager.StageFactory; - import org.apache.ambari.server.agent.CommandReport; - import org.apache.ambari.server.agent.ExecutionCommand; - import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; - import org.apache.ambari.server.api.services.AmbariMetaInfo; - import org.apache.ambari.server.configuration.Configuration; - import org.apache.ambari.server.controller.AmbariManagementController; - import org.apache.ambari.server.controller.ExecuteActionRequest; - import org.apache.ambari.server.controller.RequestStatusResponse; - import org.apache.ambari.server.controller.ResourceProviderFactory; - import org.apache.ambari.server.controller.spi.Request; - import org.apache.ambari.server.controller.spi.RequestStatus; - import org.apache.ambari.server.controller.spi.Resource; - import org.apache.ambari.server.controller.spi.ResourceProvider; - import org.apache.ambari.server.controller.utilities.PropertyHelper; - import org.apache.ambari.server.orm.GuiceJpaInitializer; - import org.apache.ambari.server.orm.InMemoryDefaultTestModule; - import org.apache.ambari.server.orm.dao.ClusterVersionDAO; - import org.apache.ambari.server.orm.dao.HostComponentStateDAO; - import org.apache.ambari.server.orm.dao.HostVersionDAO; - import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; - import org.apache.ambari.server.orm.dao.ResourceTypeDAO; - import org.apache.ambari.server.orm.dao.StackDAO; - import org.apache.ambari.server.orm.entities.ClusterEntity; - import org.apache.ambari.server.orm.entities.ClusterVersionEntity; - import org.apache.ambari.server.orm.entities.HostVersionEntity; - import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; - import org.apache.ambari.server.orm.entities.ResourceEntity; - import org.apache.ambari.server.orm.entities.ResourceTypeEntity; - import org.apache.ambari.server.orm.entities.StackEntity; - import org.apache.ambari.server.orm.entities.UpgradeEntity; - import org.apache.ambari.server.security.TestAuthenticationFactory; - import org.apache.ambari.server.security.authorization.AuthorizationException; - import org.apache.ambari.server.security.authorization.ResourceType; - import org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction; - import org.apache.ambari.server.state.Cluster; - import org.apache.ambari.server.state.Clusters; - import org.apache.ambari.server.state.ConfigHelper; - import org.apache.ambari.server.state.Host; - import org.apache.ambari.server.state.MaintenanceState; - import org.apache.ambari.server.state.RepositoryType; - import org.apache.ambari.server.state.RepositoryVersionState; - import org.apache.ambari.server.state.Service; - import org.apache.ambari.server.state.ServiceComponent; - import org.apache.ambari.server.state.ServiceComponentHost; - import org.apache.ambari.server.state.ServiceInfo; - import org.apache.ambari.server.state.ServiceOsSpecific; - import org.apache.ambari.server.state.StackId; - import org.apache.ambari.server.state.cluster.ClusterImpl; - import org.apache.ambari.server.state.stack.upgrade.Direction; - import org.apache.ambari.server.topology.TopologyManager; - import org.apache.ambari.server.utils.StageUtils; - import org.apache.commons.io.IOUtils; - import org.easymock.Capture; - import org.easymock.EasyMock; - import org.easymock.IAnswer; - import org.junit.After; - import org.junit.Assert; - import org.junit.Before; - import org.junit.Ignore; - import org.junit.Test; - import org.springframework.security.core.Authentication; - import org.springframework.security.core.context.SecurityContextHolder; - - import com.google.gson.JsonArray; - import com.google.gson.JsonObject; - import com.google.gson.JsonParser; - import com.google.inject.AbstractModule; - import com.google.inject.Guice; - import com.google.inject.Injector; - import com.google.inject.util.Modules; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.capture; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.eq; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.expectLastCall; +import static org.easymock.EasyMock.replay; +import static org.easymock.EasyMock.verify; + +import java.io.File; +import java.io.FileInputStream; +import java.lang.reflect.Field; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.H2DatabaseCleaner; +import org.apache.ambari.server.Role; +import org.apache.ambari.server.actionmanager.ActionManager; +import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper; +import org.apache.ambari.server.actionmanager.HostRoleCommand; +import org.apache.ambari.server.actionmanager.Stage; +import org.apache.ambari.server.actionmanager.StageFactory; +import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; +import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.ExecuteActionRequest; +import org.apache.ambari.server.controller.RequestStatusResponse; +import org.apache.ambari.server.controller.ResourceProviderFactory; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.RequestStatus; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.ResourceProvider; +import org.apache.ambari.server.controller.utilities.PropertyHelper; +import org.apache.ambari.server.orm.GuiceJpaInitializer; +import org.apache.ambari.server.orm.InMemoryDefaultTestModule; +import org.apache.ambari.server.orm.dao.ClusterVersionDAO; +import org.apache.ambari.server.orm.dao.HostComponentStateDAO; +import org.apache.ambari.server.orm.dao.HostVersionDAO; +import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.dao.ResourceTypeDAO; +import org.apache.ambari.server.orm.dao.StackDAO; +import org.apache.ambari.server.orm.entities.ClusterEntity; +import org.apache.ambari.server.orm.entities.ClusterVersionEntity; +import org.apache.ambari.server.orm.entities.HostVersionEntity; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.orm.entities.ResourceEntity; +import org.apache.ambari.server.orm.entities.ResourceTypeEntity; +import org.apache.ambari.server.orm.entities.StackEntity; +import org.apache.ambari.server.orm.entities.UpgradeEntity; +import org.apache.ambari.server.security.TestAuthenticationFactory; +import org.apache.ambari.server.security.authorization.AuthorizationException; +import org.apache.ambari.server.security.authorization.ResourceType; +import org.apache.ambari.server.serveraction.upgrades.FinalizeUpgradeAction; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.apache.ambari.server.state.ConfigHelper; +import org.apache.ambari.server.state.Host; +import org.apache.ambari.server.state.MaintenanceState; +import org.apache.ambari.server.state.RepositoryType; +import org.apache.ambari.server.state.RepositoryVersionState; +import org.apache.ambari.server.state.Service; +import org.apache.ambari.server.state.ServiceComponent; +import org.apache.ambari.server.state.ServiceComponentHost; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.ServiceOsSpecific; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.cluster.ClusterImpl; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; +import org.apache.ambari.server.state.stack.upgrade.Direction; +import org.apache.ambari.server.topology.TopologyManager; +import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang.StringUtils; +import org.easymock.Capture; +import org.easymock.EasyMock; +import org.easymock.IAnswer; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.util.Modules; /** @@ -222,6 +223,9 @@ public class ClusterStackVersionResourceProviderTest { repoVersion.setId(1l); repoVersion.setOperatingSystems(OS_JSON); + final String hostWithoutVersionableComponents = "host2"; + + List<Host> hostsNeedingInstallCommands = new ArrayList<>(); Map<String, Host> hostsForCluster = new HashMap<>(); int hostCount = 10; for (int i = 0; i < hostCount; i++) { @@ -236,6 +240,10 @@ public class ClusterStackVersionResourceProviderTest { replay(host); hostsForCluster.put(hostname, host); + + if (!StringUtils.equals(hostWithoutVersionableComponents, hostname)) { + hostsNeedingInstallCommands.add(host); + } } final ServiceComponentHost schDatanode = createMock(ServiceComponentHost.class); @@ -300,7 +308,7 @@ public class ClusterStackVersionResourceProviderTest { @Override public List<ServiceComponentHost> answer() throws Throwable { String hostname = (String) EasyMock.getCurrentArguments()[0]; - if (hostname.equals("host2")) { + if (hostname.equals(hostWithoutVersionableComponents)) { return schsH2; } else { return schsH1; @@ -308,6 +316,10 @@ public class ClusterStackVersionResourceProviderTest { } }).anyTimes(); + expect(cluster.transitionHostsToInstalling(anyObject(ClusterVersionEntity.class), + anyObject(RepositoryVersionEntity.class), anyObject(VersionDefinitionXml.class), + eq(false))).andReturn(hostsNeedingInstallCommands).atLeastOnce(); + ExecutionCommand executionCommand = createNiceMock(ExecutionCommand.class); ExecutionCommandWrapper executionCommandWrapper = createNiceMock(ExecutionCommandWrapper.class); @@ -644,7 +656,8 @@ public class ClusterStackVersionResourceProviderTest { ambariMetaInfo.getComponent("HDP", "2.1.1", "HBASE", "HBASE_MASTER").setVersionAdvertised(true); - + final String hostWithoutVersionableComponents = "host3"; + List<Host> hostsNeedingInstallCommands = new ArrayList<>(); Map<String, Host> hostsForCluster = new HashMap<>(); int hostCount = 10; for (int i = 0; i < hostCount; i++) { @@ -659,6 +672,11 @@ public class ClusterStackVersionResourceProviderTest { replay(host); hostsForCluster.put(hostname, host); + + + if (!StringUtils.equals(hostWithoutVersionableComponents, hostname)) { + hostsNeedingInstallCommands.add(host); + } } Service hdfsService = createNiceMock(Service.class); @@ -735,7 +753,7 @@ public class ClusterStackVersionResourceProviderTest { @Override public List<ServiceComponentHost> answer() throws Throwable { String hostname = (String) EasyMock.getCurrentArguments()[0]; - if (hostname.equals("host2")) { + if (hostname.equals("host2")) { return schsH2; } else if (hostname.equals("host3")) { return schsH3; @@ -745,6 +763,10 @@ public class ClusterStackVersionResourceProviderTest { } }).anyTimes(); + expect(cluster.transitionHostsToInstalling(anyObject(ClusterVersionEntity.class), + anyObject(RepositoryVersionEntity.class), anyObject(VersionDefinitionXml.class), + eq(false))).andReturn(hostsNeedingInstallCommands).atLeastOnce(); + // ExecutionCommand executionCommand = createNiceMock(ExecutionCommand.class); ExecutionCommand executionCommand = new ExecutionCommand(); ExecutionCommandWrapper executionCommandWrapper = createNiceMock(ExecutionCommandWrapper.class); @@ -877,6 +899,9 @@ public class ClusterStackVersionResourceProviderTest { ambariMetaInfo.getComponent("HDP", "2.1.1", "HBASE", "HBASE_MASTER").setVersionAdvertised(true); + final String hostWithoutVersionableComponents = "host3"; + List<Host> hostsNeedingInstallCommands = new ArrayList<>(); + Map<String, Host> hostsForCluster = new HashMap<>(); int hostCount = 10; for (int i = 0; i < hostCount; i++) { @@ -891,6 +916,10 @@ public class ClusterStackVersionResourceProviderTest { replay(host); hostsForCluster.put(hostname, host); + + if (!StringUtils.equals(hostWithoutVersionableComponents, hostname)) { + hostsNeedingInstallCommands.add(host); + } } Service hdfsService = createNiceMock(Service.class); @@ -977,6 +1006,10 @@ public class ClusterStackVersionResourceProviderTest { } }).anyTimes(); + expect(cluster.transitionHostsToInstalling(anyObject(ClusterVersionEntity.class), + anyObject(RepositoryVersionEntity.class), anyObject(VersionDefinitionXml.class), + eq(false))).andReturn(hostsNeedingInstallCommands).atLeastOnce(); + // ExecutionCommand executionCommand = createNiceMock(ExecutionCommand.class); ExecutionCommand executionCommand = new ExecutionCommand(); ExecutionCommandWrapper executionCommandWrapper = createNiceMock(ExecutionCommandWrapper.class); @@ -998,10 +1031,8 @@ public class ClusterStackVersionResourceProviderTest { anyObject(String.class))).andReturn(stage). times((int) Math.ceil(hostCount / MAX_TASKS_PER_STAGE)); - expect( - repositoryVersionDAOMock.findByStackAndVersion( - anyObject(StackId.class), - anyObject(String.class))).andReturn(repoVersion); + expect(repositoryVersionDAOMock.findByStackAndVersion(anyObject(StackId.class), + anyObject(String.class))).andReturn(repoVersion); Capture<org.apache.ambari.server.actionmanager.Request> c = Capture.newInstance(); Capture<ExecuteActionRequest> ear = Capture.newInstance(); @@ -1636,7 +1667,9 @@ public class ClusterStackVersionResourceProviderTest { repoVersionEntity.setVersionXml(IOUtils.toString(new FileInputStream(f))); repoVersionEntity.setVersionXsd("version_definition.xsd"); repoVersionEntity.setType(RepositoryType.STANDARD); + repoVersionEntity.setVersion(repoVersion); + List<Host> hostsNeedingInstallCommands = new ArrayList<>(); Map<String, Host> hostsForCluster = new HashMap<>(); List<HostVersionEntity> hostVersionEntitiesMergedWithNotRequired = new ArrayList<>(); int hostCount = 10; @@ -1652,6 +1685,7 @@ public class ClusterStackVersionResourceProviderTest { // transition correct into the not required state if (i < hostCount - 2) { expect(host.hasComponentsAdvertisingVersions(eq(stackId))).andReturn(true).atLeastOnce(); + hostsNeedingInstallCommands.add(host); } else { expect(host.hasComponentsAdvertisingVersions(eq(stackId))).andReturn(false).atLeastOnce(); @@ -1662,7 +1696,8 @@ public class ClusterStackVersionResourceProviderTest { replay(hostVersionEntity); hostVersionEntitiesMergedWithNotRequired.add(hostVersionEntity); - expect(host.getAllHostVersions()).andReturn(hostVersionEntitiesMergedWithNotRequired).anyTimes(); + expect(host.getAllHostVersions()).andReturn( + hostVersionEntitiesMergedWithNotRequired).anyTimes(); } replay(host); @@ -1735,15 +1770,15 @@ public class ClusterStackVersionResourceProviderTest { // then return the real one it's going to use expect(clusterVersionDAO.findByClusterAndStackAndVersion(anyObject(String.class), anyObject(StackId.class), anyObject(String.class))).andReturn(null).once(); - expect(clusterVersionDAO.findByClusterAndStackAndVersion(anyObject(String.class), - anyObject(StackId.class), anyObject(String.class))).andReturn(cve).once(); + + expect(cluster.createClusterVersion(anyObject(StackId.class), eq(repoVersion), + EasyMock.anyString(), eq(RepositoryVersionState.INSTALLED))).andReturn(cve).once(); // now the important expectations - that the cluster transition methods were // called correctly - cluster.transitionHosts(cve, RepositoryVersionState.INSTALLED); - for (HostVersionEntity hostVersionEntity : hostVersionEntitiesMergedWithNotRequired) { - expect(hostVersionDAO.merge(hostVersionEntity)).andReturn(hostVersionEntity).once(); - } + expect(cluster.transitionHostsToInstalling(cve, repoVersionEntity, + repoVersionEntity.getRepositoryXml(), true)).andReturn( + hostsNeedingInstallCommands).once(); // replay replay(managementController, response, clusters, hdfsService, resourceProviderFactory, http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java index dc9ce5e..fef9276 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/events/listeners/upgrade/HostVersionOutOfSyncListenerTest.java @@ -71,6 +71,8 @@ public class HostVersionOutOfSyncListenerTest { private final String stackId = "HDP-2.2.0"; private final String yetAnotherStackId = "HDP-2.1.1"; + private final String CURRENT_VERSION = "2.2.0-2086"; + private Injector injector; @Inject @@ -122,9 +124,6 @@ public class HostVersionOutOfSyncListenerTest { * @throws AmbariException */ private void createClusterAndHosts(String INSTALLED_VERSION, StackId stackId) throws AmbariException { - // Configuring 3-node cluster with 2 repo versions - String CURRENT_VERSION = "2.2.0-2086"; - Host h1 = clusters.getHost("h1"); h1.setState(HostState.HEALTHY); @@ -133,6 +132,10 @@ public class HostVersionOutOfSyncListenerTest { addHost("h3"); clusters.mapHostToCluster("h3", "c1"); + // create the new repo version + RepositoryVersionEntity repositoryVersionEntity = helper.getOrCreateRepositoryVersion(stackId, + INSTALLED_VERSION); + c1.createClusterVersion(stackId, INSTALLED_VERSION, "admin", RepositoryVersionState.INSTALLING); c1.setCurrentStackVersion(stackId); c1.recalculateAllClusterVersionStates(); @@ -151,13 +154,14 @@ public class HostVersionOutOfSyncListenerTest { zkTopology.put("ZOOKEEPER_SERVER", new ArrayList<>(zkServerHosts)); addService(c1, hostList, zkTopology, "ZOOKEEPER"); - // Register and install new version - RepositoryVersionEntity repositoryVersionEntity = helper.getOrCreateRepositoryVersion(stackId, - INSTALLED_VERSION); + // install new version helper.createHostVersion("h1", repositoryVersionEntity, RepositoryVersionState.INSTALLED); + helper.createHostVersion("h2", repositoryVersionEntity, RepositoryVersionState.INSTALLED); + helper.createHostVersion("h3", repositoryVersionEntity, RepositoryVersionState.INSTALLED); + c1.recalculateAllClusterVersionStates(); - assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION, - RepositoryVersionState.INSTALLED); + + assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION, RepositoryVersionState.INSTALLED); assertRepoVersionState(stackId.getStackId(), CURRENT_VERSION, RepositoryVersionState.CURRENT); // Add new host and verify that it has all host versions present @@ -210,9 +214,15 @@ public class HostVersionOutOfSyncListenerTest { StackId stackId = new StackId(this.stackId); StackId yaStackId = new StackId(yetAnotherStackId); + // get new hosts installed with the first repo createClusterAndHosts(INSTALLED_VERSION, stackId); + + // register the new repo addRepoVersion(INSTALLED_VERSION_2, yaStackId); + assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,RepositoryVersionState.INSTALLED); + assertRepoVersionState(yaStackId.getStackId(), INSTALLED_VERSION_2,RepositoryVersionState.INSTALLED); + assertRepoVersionState(yaStackId.getStackId(), CURRENT_VERSION, RepositoryVersionState.CURRENT); //Add HDFS service List<String> hostList = new ArrayList<>(); @@ -233,10 +243,6 @@ public class HostVersionOutOfSyncListenerTest { List<HostVersionEntity> hostVersions = hostVersionDAO.findAll(); - assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION, - RepositoryVersionState.INSTALLED); - assertRepoVersionState(yaStackId.getStackId(), INSTALLED_VERSION_2, - RepositoryVersionState.INSTALLED); for (HostVersionEntity hostVersionEntity : hostVersions) { if (hostVersionEntity.getRepositoryVersion().getVersion().equals(INSTALLED_VERSION) || hostVersionEntity.getRepositoryVersion().getVersion().equals(INSTALLED_VERSION_2)) { @@ -247,6 +253,11 @@ public class HostVersionOutOfSyncListenerTest { } } } + + + assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,RepositoryVersionState.OUT_OF_SYNC); + assertRepoVersionState(yaStackId.getStackId(), INSTALLED_VERSION_2,RepositoryVersionState.OUT_OF_SYNC); + assertRepoVersionState(yaStackId.getStackId(), CURRENT_VERSION, RepositoryVersionState.CURRENT); } @@ -306,6 +317,9 @@ public class HostVersionOutOfSyncListenerTest { createClusterAndHosts(INSTALLED_VERSION, stackId); addRepoVersion(INSTALLED_VERSION_2, yaStackId); + assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION, RepositoryVersionState.INSTALLED); + assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION_2, RepositoryVersionState.INSTALLED); + //Add ZOOKEEPER_CLIENT component List<String> hostList = new ArrayList<>(); hostList.add("h1"); @@ -319,8 +333,8 @@ public class HostVersionOutOfSyncListenerTest { changedHosts.add("h2"); changedHosts.add("h3"); - assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION, - RepositoryVersionState.INSTALLED); + assertRepoVersionState(stackId.getStackId(), INSTALLED_VERSION,RepositoryVersionState.OUT_OF_SYNC); + List<HostVersionEntity> hostVersions = hostVersionDAO.findAll(); for (HostVersionEntity hostVersionEntity : hostVersions) { http://git-wip-us.apache.org/repos/asf/ambari/blob/56f838b5/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java index 345c463..e5e2643 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/cluster/ClusterTest.java @@ -224,8 +224,12 @@ public class ClusterTest { } private void createDefaultCluster(Set<String> hostNames) throws Exception { - // TODO, use common function StackId stackId = new StackId("HDP", "0.1"); + createDefaultCluster(hostNames, stackId); + } + + private void createDefaultCluster(Set<String> hostNames, StackId stackId) throws Exception { + // TODO, use common function StackEntity stackEntity = stackDAO.find(stackId.getStackName(), stackId.getStackVersion()); org.junit.Assert.assertNotNull(stackEntity); @@ -1638,19 +1642,20 @@ public class ClusterTest { * Tests that hosts can be correctly transitioned into the "INSTALLING" state. * This method also tests that hosts in MM will not be transitioned, as per * the contract of - * {@link Cluster#transitionHostsToInstalling(ClusterVersionEntity)}. + * {@link Cluster#transitionHostsToInstalling(ClusterVersionEntity, RepositoryVersionEntity, org.apache.ambari.server.state.repository.VersionDefinitionXml, boolean)}. * * @throws Exception */ @Test - public void testTransitionHostVersions() throws Exception { - createDefaultCluster(); + public void testTransitionHostsToInstalling() throws Exception { + // this will create a cluster with a few hosts and no host components + StackId originalStackId = new StackId("HDP", "2.0.5"); + createDefaultCluster(Sets.newHashSet("h1", "h2"), originalStackId); - StackId stackId = new StackId("HDP", "0.2"); + StackId stackId = new StackId("HDP", "2.0.6"); helper.getOrCreateRepositoryVersion(stackId, stackId.getStackVersion()); - c1.createClusterVersion(stackId, "0.2", "admin", - RepositoryVersionState.INSTALLING); + c1.createClusterVersion(stackId, "2.0.6", "admin", RepositoryVersionState.INSTALLING); ClusterVersionEntity entityHDP2 = null; for (ClusterVersionEntity entity : c1.getAllClusterVersions()) { @@ -1658,7 +1663,7 @@ public class ClusterTest { StackId repoVersionStackId = new StackId(repoVersionStackEntity); if (repoVersionStackId.getStackName().equals("HDP") - && repoVersionStackId.getStackVersion().equals("0.2")) { + && repoVersionStackId.getStackVersion().equals("2.0.6")) { entityHDP2 = entity; break; } @@ -1669,7 +1674,9 @@ public class ClusterTest { List<HostVersionEntity> hostVersionsH1Before = hostVersionDAO.findByClusterAndHost("c1", "h1"); assertEquals(1, hostVersionsH1Before.size()); - c1.transitionHosts(entityHDP2, RepositoryVersionState.INSTALLING); + // this should move both to NOT_REQUIRED since they have no versionable + // components + c1.transitionHostsToInstalling(entityHDP2, entityHDP2.getRepositoryVersion(), null, false); List<HostVersionEntity> hostVersionsH1After = hostVersionDAO.findByClusterAndHost("c1", "h1"); assertEquals(2, hostVersionsH1After.size()); @@ -1678,8 +1685,8 @@ public class ClusterTest { for (HostVersionEntity entity : hostVersionsH1After) { StackEntity repoVersionStackEntity = entity.getRepositoryVersion().getStack(); if (repoVersionStackEntity.getStackName().equals("HDP") - && repoVersionStackEntity.getStackVersion().equals("0.2")) { - assertEquals(RepositoryVersionState.INSTALLING, entity.getState()); + && repoVersionStackEntity.getStackVersion().equals("2.0.6")) { + assertEquals(RepositoryVersionState.NOT_REQUIRED, entity.getState()); checked = true; break; } @@ -1687,8 +1694,29 @@ public class ClusterTest { assertTrue(checked); - // Test for update of existing host stack version - c1.transitionHosts(entityHDP2, RepositoryVersionState.INSTALLING); + // add some host components + Service hdfs = serviceFactory.createNew(c1, "HDFS"); + c1.addService(hdfs); + + // Add HDFS components + ServiceComponent datanode = serviceComponentFactory.createNew(hdfs, "NAMENODE"); + ServiceComponent namenode = serviceComponentFactory.createNew(hdfs, "DATANODE"); + hdfs.addServiceComponent(datanode); + hdfs.addServiceComponent(namenode); + + // add to hosts + ServiceComponentHost namenodeHost1 = serviceComponentHostFactory.createNew(namenode, "h1"); + ServiceComponentHost datanodeHost2 = serviceComponentHostFactory.createNew(datanode, "h2"); + + assertNotNull(namenodeHost1); + assertNotNull(datanodeHost2); + + c1.transitionClusterVersion(stackId, entityHDP2.getRepositoryVersion().getVersion(), + RepositoryVersionState.INSTALLING); + + // with hosts now having components which report versions, we should have + // two in the INSTALLING state + c1.transitionHostsToInstalling(entityHDP2, entityHDP2.getRepositoryVersion(), null, false); hostVersionsH1After = hostVersionDAO.findByClusterAndHost("c1", "h1"); assertEquals(2, hostVersionsH1After.size()); @@ -1697,7 +1725,7 @@ public class ClusterTest { for (HostVersionEntity entity : hostVersionsH1After) { StackEntity repoVersionStackEntity = entity.getRepositoryVersion().getStack(); if (repoVersionStackEntity.getStackName().equals("HDP") - && repoVersionStackEntity.getStackVersion().equals("0.2")) { + && repoVersionStackEntity.getStackVersion().equals("2.0.6")) { assertEquals(RepositoryVersionState.INSTALLING, entity.getState()); checked = true; break; @@ -1727,7 +1755,7 @@ public class ClusterTest { hostInMaintenanceMode.setMaintenanceState(c1.getClusterId(), MaintenanceState.ON); // transition host versions to INSTALLING - c1.transitionHosts(entityHDP2, RepositoryVersionState.INSTALLING); + c1.transitionHostsToInstalling(entityHDP2, entityHDP2.getRepositoryVersion(), null, false); List<HostVersionEntity> hostInMaintModeVersions = hostVersionDAO.findByClusterAndHost("c1", hostInMaintenanceMode.getHostName()); @@ -1739,7 +1767,7 @@ public class ClusterTest { for (HostVersionEntity hostVersionEntity : hostInMaintModeVersions) { StackEntity repoVersionStackEntity = hostVersionEntity.getRepositoryVersion().getStack(); if (repoVersionStackEntity.getStackName().equals("HDP") - && repoVersionStackEntity.getStackVersion().equals("0.2")) { + && repoVersionStackEntity.getStackVersion().equals("2.0.6")) { assertEquals(RepositoryVersionState.OUT_OF_SYNC, hostVersionEntity.getState()); } } @@ -1748,7 +1776,7 @@ public class ClusterTest { for (HostVersionEntity hostVersionEntity : otherHostVersions) { StackEntity repoVersionStackEntity = hostVersionEntity.getRepositoryVersion().getStack(); if (repoVersionStackEntity.getStackName().equals("HDP") - && repoVersionStackEntity.getStackVersion().equals("0.2")) { + && repoVersionStackEntity.getStackVersion().equals("2.0.6")) { assertEquals(RepositoryVersionState.INSTALLING, hostVersionEntity.getState()); } }