AMBARI-21175 - Host Versions Remain on CURRENT During STANDARD (Full) Upgrades (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7847e4b6 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7847e4b6 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7847e4b6 Branch: refs/heads/branch-feature-AMBARI-20859 Commit: 7847e4b6cf65b777e516a4806cbf5f45cdd207ee Parents: 4767389 Author: Jonathan Hurley <jhur...@hortonworks.com> Authored: Fri Jun 2 08:28:51 2017 -0400 Committer: Jonathan Hurley <jhur...@hortonworks.com> Committed: Fri Jun 2 10:30:06 2017 -0400 ---------------------------------------------------------------------- .../ambari/server/orm/dao/HostVersionDAO.java | 16 +++ .../server/orm/entities/HostVersionEntity.java | 5 + .../upgrades/FinalizeUpgradeAction.java | 57 ++++++++-- .../upgrades/UpgradeActionTest.java | 109 +++++++++++++++++-- 4 files changed, 164 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/7847e4b6/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java index ad34ec7..7be7691 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/dao/HostVersionDAO.java @@ -130,6 +130,22 @@ public class HostVersionDAO extends CrudDAO<HostVersionEntity, Long> { } /** + * Retrieve all of the host versions for the given cluster name, and state. <br/> + * @param clusterName Cluster name + * @param state repository version state + * @return Return all of the host versions that match the criteria. + */ + @RequiresSession + public List<HostVersionEntity> findByClusterAndState(String clusterName, RepositoryVersionState state) { + final TypedQuery<HostVersionEntity> query = entityManagerProvider.get() + .createNamedQuery("findByClusterAndState", HostVersionEntity.class); + query.setParameter("clusterName", clusterName); + query.setParameter("state", state); + + return daoUtils.selectList(query); + } + + /** * Retrieve all of the host versions for the given cluster name, host name, and state. <br/> * @param clusterName Cluster name * @param hostName FQDN of host http://git-wip-us.apache.org/repos/asf/ambari/blob/7847e4b6/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java index dbe4fe0..4a030af 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/HostVersionEntity.java @@ -61,6 +61,11 @@ import org.apache.ambari.server.state.RepositoryVersionState; "SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity host " + "WHERE hostVersion.hostEntity.hostName=:hostName"), + @NamedQuery( + name = "findByClusterAndState", + query = "SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity host JOIN host.clusterEntities clusters " + + "WHERE clusters.clusterName=:clusterName AND hostVersion.state=:state"), + @NamedQuery(name = "hostVersionByClusterHostnameAndState", query = "SELECT hostVersion FROM HostVersionEntity hostVersion JOIN hostVersion.hostEntity host JOIN host.clusterEntities clusters " + "WHERE clusters.clusterName=:clusterName AND hostVersion.hostEntity.hostName=:hostName AND hostVersion.state=:state"), http://git-wip-us.apache.org/repos/asf/ambari/blob/7847e4b6/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java index 475a8c9..8f206bf 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/FinalizeUpgradeAction.java @@ -140,8 +140,8 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction { throw new AmbariException(messageBuff.toString()); } - // for all hosts participating in this upgrade, update thei repository - // versions and upgrade state + // find every host version for this upgrade and ensure it has transitioned + // to CURRENT if required List<HostVersionEntity> hostVersions = hostVersionDAO.findHostVersionByClusterAndRepository( cluster.getClusterId(), repositoryVersion); @@ -165,7 +165,8 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction { } } - // throw an exception if there are hosts which are not not fully upgraded + // throw an exception if there are hosts which did not transition the + // repository to CURRENT if (hostsWithoutCorrectVersionState.size() > 0) { message = String.format( "The following %d host(s) have not been upgraded to version %s. " @@ -178,10 +179,11 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction { } outSB.append( - String.format("Finalizing the upgrade state of %d host(s).", + String.format("Finalizing the upgrade state and repository version for %d host(s).", hostVersionsAllowed.size())).append(System.lineSeparator()); - // Reset the upgrade state + // at this point, all host versions are correct - do some cleanup like + // resetting the upgrade state for (HostVersionEntity hostVersion : hostVersionsAllowed) { Collection<HostComponentStateEntity> hostComponentStates = hostComponentStateDAO.findByHost(hostVersion.getHostName()); for (HostComponentStateEntity hostComponentStateEntity: hostComponentStates) { @@ -190,16 +192,16 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction { } } - // Impacts all hosts that have a version - outSB.append( - String.format("Finalizing the version for %d host(s).", - hostVersionsAllowed.size())).append(System.lineSeparator()); - - versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster)); + // move host versions from CURRENT to INSTALLED if their repos are no + // longer used + finalizeHostRepositoryVersions(cluster); // Reset upgrade state cluster.setUpgradeEntity(null); + // the upgrade is done! + versionEventPublisher.publish(new StackUpgradeFinishEvent(cluster)); + message = String.format("The upgrade to %s has completed.", version); outSB.append(message).append(System.lineSeparator()); return createCommandReport(0, HostRoleStatus.COMPLETED, "{}", outSB.toString(), errSB.toString()); @@ -372,6 +374,39 @@ public class FinalizeUpgradeAction extends AbstractUpgradeServerAction { return errors; } + /** + * Moves any {@link HostVersionEntity}s which are + * {@link RepositoryVersionState#CURRENT} to either + * {@link RepositoryVersionState#INSTALLED} or + * {@link RepositoryVersionState#NOT_REQUIRED} if their assocaited + * repositories are no longer in use. + * + * @param cluster + * @throws AmbariException + */ + private void finalizeHostRepositoryVersions(Cluster cluster) throws AmbariException { + // create a set of all of the repos that the services are on + Set<RepositoryVersionEntity> desiredRepoVersions = new HashSet<>(); + Set<String> serviceNames = cluster.getServices().keySet(); + for (String serviceName : serviceNames) { + Service service = cluster.getService(serviceName); + desiredRepoVersions.add(service.getDesiredRepositoryVersion()); + } + + // if any CURRENT host version is for a repo which is no longer desired by + // ANY service, move it to INSTALLED + List<HostVersionEntity> currentHostVersions = hostVersionDAO.findByClusterAndState( + cluster.getClusterName(), RepositoryVersionState.CURRENT); + + for (HostVersionEntity hostVersion : currentHostVersions) { + RepositoryVersionEntity hostRepoVersion = hostVersion.getRepositoryVersion(); + if (!desiredRepoVersions.contains(hostRepoVersion)) { + hostVersion.setState(RepositoryVersionState.INSTALLED); + hostVersion = hostVersionDAO.merge(hostVersion); + } + } + } + protected static class InfoTuple implements Comparable<InfoTuple> { protected final String serviceName; protected final String componentName; http://git-wip-us.apache.org/repos/asf/ambari/blob/7847e4b6/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java index 35fffda..e3ffae0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java @@ -100,7 +100,6 @@ public class UpgradeActionTest { private static final String HDP_2_1_1_1 = "2.1.1.1-2"; private static final String HDP_2_2_0_1 = "2.2.0.1-3"; - private static final String HDP_2_2_0_2 = "2.2.0.2-4"; private static final StackId HDP_21_STACK = new StackId("HDP-2.1.1"); private static final StackId HDP_22_STACK = new StackId("HDP-2.2.0"); @@ -147,7 +146,6 @@ public class UpgradeActionTest { private RepositoryVersionEntity repositoryVersion2110; private RepositoryVersionEntity repositoryVersion2111; private RepositoryVersionEntity repositoryVersion2201; - private RepositoryVersionEntity repositoryVersion2202; @Before public void setup() throws Exception { @@ -167,7 +165,6 @@ public class UpgradeActionTest { repositoryVersion2110 = m_helper.getOrCreateRepositoryVersion(HDP_21_STACK, HDP_2_1_1_0); repositoryVersion2111 = m_helper.getOrCreateRepositoryVersion(HDP_21_STACK, HDP_2_1_1_1); repositoryVersion2201 = m_helper.getOrCreateRepositoryVersion(HDP_22_STACK, HDP_2_2_0_1); - repositoryVersion2202 = m_helper.getOrCreateRepositoryVersion(HDP_22_STACK, HDP_2_2_0_2); } @After @@ -199,11 +196,11 @@ public class UpgradeActionTest { hostVersionDAO.create(entity); } - private void createUpgradeCluster( + private Cluster createUpgradeCluster( RepositoryVersionEntity sourceRepoVersion, String hostName) throws Exception { clusters.addCluster(clusterName, sourceRepoVersion.getStackId()); - Cluster c = clusters.getCluster(clusterName); + Cluster cluster = clusters.getCluster(clusterName); // add a host component clusters.addHost(hostName); @@ -222,6 +219,8 @@ public class UpgradeActionTest { sourceRepoVersion, RepositoryVersionState.INSTALLED); hostVersionDAO.create(entity); + + return cluster; } private void createHostVersions(RepositoryVersionEntity targetRepoVersion, @@ -313,13 +312,13 @@ public class UpgradeActionTest { Cluster cluster = clusters.getCluster(clusterName); // Install ZK and HDFS with some components - Service zk = installService(cluster, "ZOOKEEPER"); + Service zk = installService(cluster, "ZOOKEEPER", repositoryVersion2110); addServiceComponent(cluster, zk, "ZOOKEEPER_SERVER"); addServiceComponent(cluster, zk, "ZOOKEEPER_CLIENT"); createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_SERVER", "h1"); createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_CLIENT", "h1"); - Service hdfs = installService(cluster, "HDFS"); + Service hdfs = installService(cluster, "HDFS", repositoryVersion2110); addServiceComponent(cluster, hdfs, "NAMENODE"); addServiceComponent(cluster, hdfs, "DATANODE"); createNewServiceComponentHost(cluster, "HDFS", "NAMENODE", "h1"); @@ -393,7 +392,7 @@ public class UpgradeActionTest { for (HostVersionEntity entity : hostVersionDAO.findByClusterAndHost(clusterName, "h1")) { if (StringUtils.equals(entity.getRepositoryVersion().getVersion(), repositoryVersion2110.getVersion())) { assertEquals(RepositoryVersionState.CURRENT, entity.getState()); - } else if (StringUtils.equals(entity.getRepositoryVersion().getVersion(), repositoryVersion2111.getVersion())) { + } else { assertEquals(RepositoryVersionState.INSTALLED, entity.getState()); } } @@ -497,10 +496,95 @@ public class UpgradeActionTest { assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); } + /** + * Tests that all host versions are correct after upgrade. This test will + * ensure that the prior CURRENT versions are moved to INSTALLED while not + * touching any others. + * + * @throws Exception + */ + @Test + public void testHostVersionsAfterUpgrade() throws Exception { + String hostName = "h1"; + Cluster cluster = createUpgradeCluster(repositoryVersion2110, hostName); + createHostVersions(repositoryVersion2111, hostName); + createHostVersions(repositoryVersion2201, hostName); + + // Install ZK with some components + Service zk = installService(cluster, "ZOOKEEPER", repositoryVersion2110); + addServiceComponent(cluster, zk, "ZOOKEEPER_SERVER"); + addServiceComponent(cluster, zk, "ZOOKEEPER_CLIENT"); + createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_SERVER", hostName); + createNewServiceComponentHost(cluster, "ZOOKEEPER", "ZOOKEEPER_CLIENT", hostName); + + List<HostVersionEntity> hostVersions = hostVersionDAO.findAll(); + assertEquals(3, hostVersions.size()); + + // repo 2110 - CURRENT (upgrading from) + // repo 2111 - CURRENT (all hosts reported in during upgrade) + // repo 2201 - NOT_REQUIRED (different stack) + for (HostVersionEntity hostVersion : hostVersions) { + RepositoryVersionEntity hostRepoVersion = hostVersion.getRepositoryVersion(); + if (repositoryVersion2110.equals(hostRepoVersion)) { + hostVersion.setState(RepositoryVersionState.CURRENT); + } else if (repositoryVersion2111.equals(hostRepoVersion)) { + hostVersion.setState(RepositoryVersionState.CURRENT); + } else { + hostVersion.setState(RepositoryVersionState.NOT_REQUIRED); + } + + hostVersionDAO.merge(hostVersion); + } + + // upgrade to 2111 + createUpgrade(cluster, repositoryVersion2111); + + // push all services to the correct repo version for finalize + Map<String, Service> services = cluster.getServices(); + assertTrue(services.size() > 0); + for (Service service : services.values()) { + service.setDesiredRepositoryVersion(repositoryVersion2111); + } + + // push all components to the correct version + List<HostComponentStateEntity> hostComponentStates = hostComponentStateDAO.findByHost(hostName); + for (HostComponentStateEntity hostComponentState : hostComponentStates) { + hostComponentState.setVersion(repositoryVersion2111.getVersion()); + hostComponentStateDAO.merge(hostComponentState); + } + + Map<String, String> commandParams = new HashMap<>(); + ExecutionCommand executionCommand = new ExecutionCommand(); + executionCommand.setCommandParams(commandParams); + executionCommand.setClusterName(clusterName); + + HostRoleCommand hostRoleCommand = hostRoleCommandFactory.create(null, null, null, null); + hostRoleCommand.setExecutionCommandWrapper(new ExecutionCommandWrapper(executionCommand)); + + finalizeUpgradeAction.setExecutionCommand(executionCommand); + finalizeUpgradeAction.setHostRoleCommand(hostRoleCommand); + + // finalize + CommandReport report = finalizeUpgradeAction.execute(null); + assertNotNull(report); + assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); + + for (HostVersionEntity hostVersion : hostVersions) { + RepositoryVersionEntity hostRepoVersion = hostVersion.getRepositoryVersion(); + if (repositoryVersion2110.equals(hostRepoVersion)) { + assertEquals(RepositoryVersionState.INSTALLED, hostVersion.getState()); + } else if (repositoryVersion2111.equals(hostRepoVersion)) { + assertEquals(RepositoryVersionState.CURRENT, hostVersion.getState()); + } else { + assertEquals(RepositoryVersionState.NOT_REQUIRED, hostVersion.getState()); + } + } + } + private ServiceComponentHost createNewServiceComponentHost(Cluster cluster, String svc, String svcComponent, String hostName) throws AmbariException { Assert.assertNotNull(cluster.getConfigGroups()); - Service s = installService(cluster, svc); + Service s = installService(cluster, svc, sourceRepositoryVersion); ServiceComponent sc = addServiceComponent(cluster, s, svcComponent); ServiceComponentHost sch = serviceComponentHostFactory.createNew(sc, hostName); @@ -511,13 +595,14 @@ public class UpgradeActionTest { return sch; } - private Service installService(Cluster cluster, String serviceName) throws AmbariException { + private Service installService(Cluster cluster, String serviceName, + RepositoryVersionEntity repositoryVersionEntity) throws AmbariException { Service service = null; try { service = cluster.getService(serviceName); } catch (ServiceNotFoundException e) { - service = serviceFactory.createNew(cluster, serviceName, sourceRepositoryVersion); + service = serviceFactory.createNew(cluster, serviceName, repositoryVersionEntity); cluster.addService(service); } @@ -525,7 +610,7 @@ public class UpgradeActionTest { } private ServiceComponent addServiceComponent(Cluster cluster, Service service, - String componentName) throws AmbariException { + String componentName) throws AmbariException { ServiceComponent serviceComponent = null; try { serviceComponent = service.getServiceComponent(componentName);
