Repository: ambari Updated Branches: refs/heads/branch-feature-AMBARI-12556 d84abbfd3 -> 8782cf691
AMBARI-20923. Repositories must be resolved correctly when installing new components (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8782cf69 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8782cf69 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8782cf69 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: 8782cf69160dae3d3894d7d93a6eb3f2c384e237 Parents: d84abbf Author: Nate Cole <nc...@hortonworks.com> Authored: Wed May 3 13:28:13 2017 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Wed May 3 15:28:40 2017 -0400 ---------------------------------------------------------------------- .../libraries/script/script.py | 20 +++-- .../ambari/server/agent/CommandRepository.java | 14 ++++ .../ambari/server/agent/ExecutionCommand.java | 6 ++ .../AmbariCustomCommandExecutionHelper.java | 83 +++++++++++++------- .../AmbariManagementControllerImpl.java | 6 +- .../ServiceComponentDesiredStateEntity.java | 16 ++-- .../ambari/server/topology/AmbariContext.java | 4 +- .../AmbariCustomCommandExecutionHelperTest.java | 74 ++++++++++++++++- .../upgrades/UpgradeActionTest.java | 42 +--------- .../custom_actions/TestInstallPackages.py | 70 +++++++++++++++++ 10 files changed, 249 insertions(+), 86 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index 0dd9c02..75a1dd4 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -463,14 +463,22 @@ class Script(object): version is passed from the server, use that as an absolute truth. """ - # two different command types put things in different objects. WHY. - # package_version is the form W_X_Y_Z_nnnn - package_version = default("roleParams/package_version", None) - if not package_version: - package_version = default("hostLevelParams/package_version", None) - package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_' + # repositoryFile is the truth + # package_version should be made to the form W_X_Y_Z_nnnn + package_version = default("repositoryFile/repoVersion", None) + if package_version is not None: + package_version = package_version.replace('.', package_delimiter).replace('-', package_delimiter) + + # TODO remove legacy checks + if package_version is None: + package_version = default("roleParams/package_version", None) + + # TODO remove legacy checks + if package_version is None: + package_version = default("hostLevelParams/package_version", None) + # The cluster effective version comes down when the version is known after the initial # install. In that case we should not be guessing which version when invoking INSTALL, but # use the supplied version to build the package_version http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java index 849d6fb..3d96122 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.ambari.server.orm.entities.RepositoryEntity; import org.apache.ambari.server.state.RepositoryInfo; +import org.apache.commons.lang.builder.ToStringBuilder; import com.google.gson.annotations.SerializedName; @@ -165,6 +166,19 @@ public class CommandRepository { public String getBaseUrl() { return m_baseUrl; } + + /** + * {@inheritDoc} + */ + @Override + public String toString() { + return new ToStringBuilder(null) + .append("os", m_osType) + .append("name", m_repoName) + .append("id", m_repoId) + .append("baseUrl", m_baseUrl) + .toString(); + } } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index bd62cbb..63eb660 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -431,6 +431,8 @@ public class ExecutionCommand extends AgentCommand { String STACK_NAME = "stack_name"; String SERVICE_TYPE = "service_type"; String STACK_VERSION = "stack_version"; + @Deprecated + @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) String SERVICE_REPO_INFO = "service_repo_info"; String PACKAGE_LIST = "package_list"; String JDK_LOCATION = "jdk_location"; @@ -481,6 +483,8 @@ public class ExecutionCommand extends AgentCommand { /** * The key indicating that the package_version string is available */ + @Deprecated + @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) String PACKAGE_VERSION = "package_version"; /** @@ -495,6 +499,8 @@ public class ExecutionCommand extends AgentCommand { * The agent will return this value back in its response so the repository * can be looked up and possibly have its version updated. */ + @Deprecated + @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) String REPO_VERSION_ID = "repository_version_id"; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index d6905fb..1d43093 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -78,6 +78,7 @@ import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.metadata.ActionMetadata; import org.apache.ambari.server.orm.dao.ClusterVersionDAO; import org.apache.ambari.server.orm.dao.HostRoleCommandDAO; +import org.apache.ambari.server.orm.dao.ServiceComponentDesiredStateDAO; import org.apache.ambari.server.orm.entities.ClusterVersionEntity; import org.apache.ambari.server.orm.entities.OperatingSystemEntity; import org.apache.ambari.server.orm.entities.RepositoryEntity; @@ -182,6 +183,10 @@ public class AmbariCustomCommandExecutionHelper { @Inject private HostRoleCommandDAO hostRoleCommandDAO; + @Inject + private ServiceComponentDesiredStateDAO serviceComponentDAO; + + private Map<String, Map<String, Map<String, String>>> configCredentialsForService = new HashMap<>(); protected static final String SERVICE_CHECK_COMMAND_NAME = "SERVICE_CHECK"; @@ -399,6 +404,11 @@ public class AmbariCustomCommandExecutionHelper { Service clusterService = cluster.getService(serviceName); execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled())); + ServiceComponent component = null; + if (StringUtils.isNotBlank(componentName)) { + component = clusterService.getServiceComponent(componentName); + } + // Get the map of service config type to password properties for the service Map<String, Map<String, String>> configCredentials; configCredentials = configCredentialsForService.get(clusterService.getName()); @@ -414,7 +424,7 @@ public class AmbariCustomCommandExecutionHelper { hostLevelParams.put(CUSTOM_COMMAND, commandName); // Set parameters required for re-installing clients on restart - hostLevelParams.put(REPO_INFO, getRepoInfo(cluster, host)); + hostLevelParams.put(REPO_INFO, getRepoInfo(cluster, component, host)); hostLevelParams.put(STACK_NAME, stackId.getStackName()); hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); @@ -504,7 +514,7 @@ public class AmbariCustomCommandExecutionHelper { execCmd.setCommandParams(commandParams); execCmd.setRoleParams(roleParams); - execCmd.setRepositoryFile(getCommandRepository(cluster, host)); + execCmd.setRepositoryFile(getCommandRepository(cluster, component, host)); // perform any server side command related logic - eg - set desired states on restart applyCustomCommandBackendLogic(cluster, serviceName, componentName, commandName, hostName); @@ -1179,7 +1189,7 @@ public class AmbariCustomCommandExecutionHelper { * @throws AmbariException if the repository information can not be obtained */ @Deprecated - public String getRepoInfo(Cluster cluster, Host host) throws AmbariException { + public String getRepoInfo(Cluster cluster, ServiceComponent component, Host host) throws AmbariException { Function<List<RepositoryInfo>, JsonArray> function = new Function<List<RepositoryInfo>, JsonArray>() { @Override @@ -1188,7 +1198,7 @@ public class AmbariCustomCommandExecutionHelper { } }; - final JsonArray gsonList = getBaseUrls(cluster, host, function); + final JsonArray gsonList = getBaseUrls(cluster, component, host, function); if (null == gsonList) { return ""; @@ -1216,7 +1226,6 @@ public class AmbariCustomCommandExecutionHelper { if (ose.getOsType().equals(osType) && ose.isAmbariManagedRepos()) { for (RepositoryEntity re : ose.getRepositories()) { if (re.getName().equals(repoName) && - re.getRepositoryId().equals(repoId) && !re.getBaseUrl().equals(baseUrl)) { obj.addProperty("baseUrl", re.getBaseUrl()); } @@ -1230,7 +1239,7 @@ public class AmbariCustomCommandExecutionHelper { } }; - return updateBaseUrls(cluster, updater).toString(); + return updateBaseUrls(cluster, component, updater).toString(); } /** @@ -1243,7 +1252,7 @@ public class AmbariCustomCommandExecutionHelper { * @throws AmbariException */ @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) - public CommandRepository getCommandRepository(final Cluster cluster, Host host) throws AmbariException { + public CommandRepository getCommandRepository(final Cluster cluster, ServiceComponent component, Host host) throws AmbariException { Function<List<RepositoryInfo>, List<RepositoryInfo>> function = new Function<List<RepositoryInfo>, List<RepositoryInfo>>() { @Override @@ -1253,7 +1262,7 @@ public class AmbariCustomCommandExecutionHelper { } }; - final List<RepositoryInfo> repoInfos = getBaseUrls(cluster, host, function); + final List<RepositoryInfo> repoInfos = getBaseUrls(cluster, component, host, function); if (null == repoInfos) { return null; @@ -1275,7 +1284,6 @@ public class AmbariCustomCommandExecutionHelper { for (CommandRepository.Repository commandRepo : command.getRepositories()) { String osType = commandRepo.getOsType(); - String repoId = commandRepo.getRepoId(); String repoName = commandRepo.getRepoName(); String baseUrl = commandRepo.getBaseUrl(); @@ -1283,7 +1291,6 @@ public class AmbariCustomCommandExecutionHelper { if (ose.getOsType().equals(osType) && ose.isAmbariManagedRepos()) { for (RepositoryEntity re : ose.getRepositories()) { if (re.getName().equals(repoName) && - re.getRepositoryId().equals(repoId) && !re.getBaseUrl().equals(baseUrl)) { commandRepo.setBaseUrl(re.getBaseUrl()); } @@ -1296,7 +1303,7 @@ public class AmbariCustomCommandExecutionHelper { } }; - updateBaseUrls(cluster, updater); + updateBaseUrls(cluster, component, updater); return command; } @@ -1306,13 +1313,15 @@ public class AmbariCustomCommandExecutionHelper { * implemenation, this may be removed and called inline in {@link #getCommandRepository(Cluster, Host)} * * @param cluster the cluster to isolate the stack + * @param component the component * @param host used to resolve the family for the repositories * @param function function that will transform the supplied repositories for specific use. * @return <T> the type as defined by the supplied {@code function}. * @throws AmbariException */ @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES) - private <T> T getBaseUrls(Cluster cluster, Host host, Function<List<RepositoryInfo>, T> function) throws AmbariException { + private <T> T getBaseUrls(Cluster cluster, ServiceComponent component, Host host, + Function<List<RepositoryInfo>, T> function) throws AmbariException { String hostOsType = host.getOsType(); String hostOsFamily = host.getOsFamily(); @@ -1354,31 +1363,51 @@ public class AmbariCustomCommandExecutionHelper { * @param <T> the result after appling the repository version, if found. */ @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES) - private <T> T updateBaseUrls(Cluster cluster, BaseUrlUpdater<T> function) throws AmbariException { - ClusterVersionEntity cve = cluster.getCurrentClusterVersion(); + private <T> T updateBaseUrls(Cluster cluster, ServiceComponent component, BaseUrlUpdater<T> function) throws AmbariException { - if (null == cve) { - List<ClusterVersionEntity> list = clusterVersionDAO.findByClusterAndState(cluster.getClusterName(), - RepositoryVersionState.INIT); + RepositoryVersionEntity repositoryEntity = null; - if (!list.isEmpty()) { - if (list.size() > 1) { - throw new AmbariException(String.format("The cluster can only be initialized by one version: %s found", - list.size())); - } else { - cve = list.get(0); + // !!! try to find the component repo first + if (null != component) { + repositoryEntity = component.getDesiredRepositoryVersion(); + } + + if (null == component) { + LOG.info("Service component not passed in, attempt to resolve the repository for cluster {}", + cluster.getClusterName()); + } + + if (null == repositoryEntity) { + + ClusterVersionEntity cve = cluster.getCurrentClusterVersion(); + + if (null == cve) { + List<ClusterVersionEntity> list = clusterVersionDAO.findByClusterAndState(cluster.getClusterName(), + RepositoryVersionState.INIT); + + if (!list.isEmpty()) { + if (list.size() > 1) { + throw new AmbariException(String.format("The cluster can only be initialized by one version: %s found", + list.size())); + } else { + cve = list.get(0); + } } } + + if (null != cve && null != cve.getRepositoryVersion()) { + repositoryEntity = cve.getRepositoryVersion(); + } else { + LOG.info("Cluster {} has no specific Repository Versions. Using stack-defined values", cluster.getClusterName()); + } } - if (null == cve || null == cve.getRepositoryVersion()) { + if (null == repositoryEntity) { LOG.info("Cluster {} has no specific Repository Versions. Using stack-defined values", cluster.getClusterName()); return function.getDefault(); } - RepositoryVersionEntity rve = cve.getRepositoryVersion(); - - return function.apply(rve); + return function.apply(repositoryEntity); } http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 8995e51..67ae5d5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -2232,6 +2232,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle Service clusterService = cluster.getService(serviceName); execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled())); + ServiceComponent component = clusterService.getServiceComponent(componentName); + // Get the map of service config type to password properties for the service Map<String, Map<String, String>> configCredentials; configCredentials = configCredentialsForService.get(clusterService.getName()); @@ -2354,7 +2356,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True"); } - String repoInfo = customCommandExecutionHelper.getRepoInfo(cluster, host); + String repoInfo = customCommandExecutionHelper.getRepoInfo(cluster, component, host); if (LOG.isDebugEnabled()) { LOG.debug("Sending repo information to agent" + ", hostname=" + scHost.getHostName() @@ -2470,7 +2472,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle execCmd.setCommandParams(commandParams); execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion())); - execCmd.setRepositoryFile(customCommandExecutionHelper.getCommandRepository(cluster, host)); + execCmd.setRepositoryFile(customCommandExecutionHelper.getCommandRepository(cluster, component, host)); if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) { LOG.debug("AmbariManagementControllerImpl.createHostAction: created ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, with cluster-env tags {}", http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java index eb1b187..6b89c02 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/ServiceComponentDesiredStateEntity.java @@ -123,7 +123,7 @@ public class ServiceComponentDesiredStateEntity { private Collection<ServiceComponentHistoryEntity> serviceComponentHistory; @OneToMany(mappedBy = "m_serviceComponentDesiredStateEntity", cascade = { CascadeType.ALL }) - private Collection<ServiceComponentVersionEntity> serviceComponentVersion; + private Collection<ServiceComponentVersionEntity> serviceComponentVersions; public Long getId() { return id; @@ -205,22 +205,22 @@ public class ServiceComponentDesiredStateEntity { /** - * @param versionEntry the version to add + * @param versionEntity the version to add */ - public void addVersion(ServiceComponentVersionEntity versionEntry) { - if (null == serviceComponentVersion) { - serviceComponentVersion = new ArrayList<>(); + public void addVersion(ServiceComponentVersionEntity versionEntity) { + if (null == serviceComponentVersions) { + serviceComponentVersions = new ArrayList<>(); } - serviceComponentVersion.add(versionEntry); - versionEntry.setServiceComponentDesiredState(this); + serviceComponentVersions.add(versionEntity); + versionEntity.setServiceComponentDesiredState(this); } /** * @return the collection of versions for the component */ public Collection<ServiceComponentVersionEntity> getVersions() { - return serviceComponentVersion; + return serviceComponentVersions; } http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java index 4a1e61f..8ae8e54 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/AmbariContext.java @@ -212,8 +212,8 @@ public class AmbariContext { Set<ServiceComponentRequest> componentRequests = new HashSet<>(); for (String service : services) { String credentialStoreEnabled = topology.getBlueprint().getCredentialStoreEnabled(service); - serviceRequests.add(new ServiceRequest(clusterName, service, null, stackId.getStackId(), - repositoryVersion, credentialStoreEnabled)); + serviceRequests.add(new ServiceRequest(clusterName, service, stackId.getStackId(), + repositoryVersion, null, credentialStoreEnabled)); for (String component : topology.getBlueprint().getComponents(service)) { String recoveryEnabled = topology.getBlueprint().getRecoveryEnabled(service, component); http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java index b0d085b..d558c15 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java @@ -37,6 +37,7 @@ import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.actionmanager.Request; import org.apache.ambari.server.actionmanager.Stage; +import org.apache.ambari.server.agent.CommandRepository; import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.internal.ComponentResourceProviderTest; @@ -49,7 +50,12 @@ import org.apache.ambari.server.orm.GuiceJpaInitializer; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; import org.apache.ambari.server.orm.OrmTestHelper; import org.apache.ambari.server.orm.dao.RepositoryVersionDAO; +import org.apache.ambari.server.orm.dao.ServiceComponentDesiredStateDAO; +import org.apache.ambari.server.orm.dao.StackDAO; import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; +import org.apache.ambari.server.orm.entities.ServiceComponentDesiredStateEntity; +import org.apache.ambari.server.orm.entities.ServiceComponentVersionEntity; +import org.apache.ambari.server.orm.entities.StackEntity; import org.apache.ambari.server.security.TestAuthenticationFactory; import org.apache.ambari.server.security.authorization.AuthorizationException; import org.apache.ambari.server.state.Cluster; @@ -60,6 +66,7 @@ import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.HostState; import org.apache.ambari.server.state.MaintenanceState; import org.apache.ambari.server.state.PropertyInfo; +import org.apache.ambari.server.state.RepositoryInfo; import org.apache.ambari.server.state.SecurityType; import org.apache.ambari.server.state.Service; import org.apache.ambari.server.state.ServiceComponent; @@ -69,6 +76,7 @@ import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.State; import org.apache.ambari.server.state.UserGroupInfo; import org.apache.ambari.server.state.ValueAttributesInfo; +import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; import org.apache.ambari.server.topology.TopologyManager; import org.apache.ambari.server.utils.StageUtils; import org.easymock.Capture; @@ -555,8 +563,72 @@ public class AmbariCustomCommandExecutionHelperTest { } } + @Test + public void testCommandRepository() throws Exception { + Cluster cluster = clusters.getCluster("c1"); + Service serviceYARN = cluster.getService("YARN"); + Service serviceZK = cluster.getService("ZOOKEEPER"); + ServiceComponent componentRM = serviceYARN.getServiceComponent("RESOURCEMANAGER"); + ServiceComponent componentZKC = serviceZK.getServiceComponent("ZOOKEEPER_CLIENT"); + Host host = clusters.getHost("c1-c6401"); + + AmbariCustomCommandExecutionHelper helper = injector.getInstance(AmbariCustomCommandExecutionHelper.class); + StackDAO stackDAO = injector.getInstance(StackDAO.class); + RepositoryVersionDAO repoVersionDAO = injector.getInstance(RepositoryVersionDAO.class); + ServiceComponentDesiredStateDAO componentDAO = injector.getInstance(ServiceComponentDesiredStateDAO.class); + RepositoryVersionHelper repoVersionHelper = injector.getInstance(RepositoryVersionHelper.class); + + CommandRepository commandRepo = helper.getCommandRepository(cluster, componentRM, host); + + Assert.assertEquals(1, commandRepo.getRepositories().size()); + CommandRepository.Repository repo = commandRepo.getRepositories().iterator().next(); + Assert.assertEquals("http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0", repo.getBaseUrl()); + + RepositoryInfo ri = new RepositoryInfo(); + ri.setBaseUrl("http://foo"); + ri.setRepoName("HDP"); + ri.setRepoId("new-id"); + ri.setOsType("redhat6"); + String operatingSystems = repoVersionHelper.serializeOperatingSystems(Collections.singletonList(ri)); + + + StackEntity stackEntity = stackDAO.find(cluster.getDesiredStackVersion().getStackName(), + cluster.getDesiredStackVersion().getStackVersion()); + + RepositoryVersionEntity repositoryVersion = new RepositoryVersionEntity(stackEntity, + "2.1.1.1-1234", "2.1.1.1-1234", operatingSystems); + repositoryVersion = repoVersionDAO.merge(repositoryVersion); + + // add a repo version associated with a component + ServiceComponentDesiredStateEntity componentEntity = componentDAO.findByName(cluster.getClusterId(), + serviceYARN.getName(), componentRM.getName()); + + ServiceComponentVersionEntity componentVersionEntity = new ServiceComponentVersionEntity(); + componentVersionEntity.setRepositoryVersion(repositoryVersion); + componentVersionEntity.setUserName("admin"); + + componentEntity.setDesiredRepositoryVersion(repositoryVersion); + componentEntity.addVersion(componentVersionEntity); + componentEntity = componentDAO.merge(componentEntity); + + // !!! make sure the override is set + commandRepo = helper.getCommandRepository(cluster, componentRM, host); + + Assert.assertEquals(1, commandRepo.getRepositories().size()); + repo = commandRepo.getRepositories().iterator().next(); + Assert.assertEquals("http://foo", repo.getBaseUrl()); + + // verify that ZK is NOT overwritten + commandRepo = helper.getCommandRepository(cluster, componentZKC, host); + + Assert.assertEquals(1, commandRepo.getRepositories().size()); + repo = commandRepo.getRepositories().iterator().next(); + Assert.assertEquals("http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0", repo.getBaseUrl()); + } + private void createClusterFixture(String clusterName, StackId stackId, - String respositoryVersion, String hostPrefix) throws AmbariException, AuthorizationException { + String respositoryVersion, String hostPrefix) throws AmbariException, AuthorizationException { + String hostC6401 = hostPrefix + "-c6401"; String hostC6402 = hostPrefix + "-c6402"; http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java index 0a583ae..18eef56 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java @@ -39,10 +39,8 @@ import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.agent.CommandReport; -import org.apache.ambari.server.agent.CommandRepository; import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.api.services.AmbariMetaInfo; -import org.apache.ambari.server.controller.AmbariCustomCommandExecutionHelper; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; import org.apache.ambari.server.controller.ServiceConfigVersionResponse; @@ -89,10 +87,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import com.google.gson.Gson; -import com.google.gson.JsonArray; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; import com.google.inject.Guice; import com.google.inject.Inject; import com.google.inject.Injector; @@ -320,7 +314,7 @@ public class UpgradeActionTest { // Create the new repo version String urlInfo = "[{'repositories':[" - + "{'Repositories/base_url':'http://foo1','Repositories/repo_name':'HDP','Repositories/repo_id':'" + targetStack.getStackId() + "'}" + + "{'Repositories/base_url':'http://foo1','Repositories/repo_name':'HDP','Repositories/repo_id':'" + targetStack.getStackId() + "-1'}" + "], 'OperatingSystems/os_type':'redhat6'}]"; repoVersionDAO.create(stackEntityTarget, targetRepo, String.valueOf(System.currentTimeMillis()), urlInfo); @@ -604,13 +598,8 @@ public class UpgradeActionTest { createUpgrade(cluster, sourceStack, sourceRepo, targetRepo); - // Verify the repo before calling Finalize - AmbariCustomCommandExecutionHelper helper = m_injector.getInstance(AmbariCustomCommandExecutionHelper.class); - Host host = clusters.getHost("h1"); - RepositoryInfo repo = ambariMetaInfo.getRepository(sourceStack.getStackName(), sourceStack.getStackVersion(), "redhat6", sourceStack.getStackId()); assertEquals(HDP_211_CENTOS6_REPO_URL, repo.getBaseUrl()); - verifyBaseRepoURL(helper, cluster, host, HDP_211_CENTOS6_REPO_URL); // Finalize the upgrade Map<String, String> commandParams = new HashMap<>(); @@ -664,8 +653,6 @@ public class UpgradeActionTest { } // Verify the repo before calling Finalize - AmbariCustomCommandExecutionHelper helper = m_injector.getInstance(AmbariCustomCommandExecutionHelper.class); - Host host = clusters.getHost("h1"); Cluster cluster = clusters.getCluster(clusterName); createUpgrade(cluster, sourceStack, sourceRepo, targetRepo); @@ -673,7 +660,6 @@ public class UpgradeActionTest { RepositoryInfo repo = ambariMetaInfo.getRepository(sourceStack.getStackName(), sourceStack.getStackVersion(), "redhat6", sourceStack.getStackId()); assertEquals(HDP_211_CENTOS6_REPO_URL, repo.getBaseUrl()); - verifyBaseRepoURL(helper, cluster, host, HDP_211_CENTOS6_REPO_URL); // Finalize the upgrade Map<String, String> commandParams = new HashMap<>(); @@ -695,27 +681,6 @@ public class UpgradeActionTest { assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); } - private void verifyBaseRepoURL(AmbariCustomCommandExecutionHelper helper, Cluster cluster, Host host, String expectedRepoBaseURL) throws AmbariException { - - String repoInfo = helper.getRepoInfo(cluster, host); - Gson gson = new Gson(); - JsonElement element = gson.fromJson(repoInfo, JsonElement.class); - assertTrue(element.isJsonArray()); - JsonArray list = JsonArray.class.cast(element); - assertEquals(1, list.size()); - - JsonObject o = list.get(0).getAsJsonObject(); - assertTrue(o.has("baseUrl")); - assertEquals(expectedRepoBaseURL, o.get("baseUrl").getAsString()); - - CommandRepository commandRepo = helper.getCommandRepository(cluster, host); - - assertNotNull(commandRepo); - assertNotNull(commandRepo.getRepositories()); - assertEquals(1, commandRepo.getRepositories().size()); - assertEquals(expectedRepoBaseURL, commandRepo.getRepositories().iterator().next().getBaseUrl()); - } - @Test public void testFinalizeUpgradeAcrossStacks() throws Exception { StackId sourceStack = HDP_21_STACK; @@ -958,9 +923,6 @@ public class UpgradeActionTest { createUpgradeClusterAndSourceRepo(sourceStack, sourceRepo, hostName); - // Verify the repo before calling Finalize - AmbariCustomCommandExecutionHelper helper = m_injector.getInstance(AmbariCustomCommandExecutionHelper.class); - Host host = clusters.getHost("h1"); Cluster cluster = clusters.getCluster(clusterName); // install HDFS with some components @@ -989,7 +951,7 @@ public class UpgradeActionTest { RepositoryInfo repo = ambariMetaInfo.getRepository(sourceStack.getStackName(), sourceStack.getStackVersion(), "redhat6", sourceStack.getStackId()); assertEquals(HDP_211_CENTOS6_REPO_URL, repo.getBaseUrl()); - verifyBaseRepoURL(helper, cluster, host, HDP_211_CENTOS6_REPO_URL); +// verifyBaseRepoURL(helper, cluster, null, host, HDP_211_CENTOS6_REPO_URL); // Finalize the upgrade, passing in the request ID so that history is // created http://git-wip-us.apache.org/repos/asf/ambari/blob/8782cf69/ambari-server/src/test/python/custom_actions/TestInstallPackages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py index 5206ea3..5f5576e 100644 --- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py +++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py @@ -492,6 +492,76 @@ class TestInstallPackages(RMFTestCase): self.assertNoMoreResources() + @patch("ambari_commons.os_check.OSCheck.is_suse_family") + @patch("resource_management.core.resources.packaging.Package") + @patch("resource_management.libraries.script.Script.put_structured_out") + @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.functions.stack_select.get_stack_versions") + @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") + @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") + def test_format_package_name_via_repositoryFile(self, write_actual_version_to_history_file_mock, + read_actual_version_from_history_file_mock, + stack_versions_mock, + allInstalledPackages_mock, put_structured_out_mock, + package_mock, is_suse_family_mock): + Script.stack_version_from_distro_select = VERSION_STUB + stack_versions_mock.side_effect = [ + [], # before installation attempt + [VERSION_STUB] + ] + read_actual_version_from_history_file_mock.return_value = VERSION_STUB + allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages) + is_suse_family_mock.return_value = True + + + config_file = self.get_src_folder() + "/test/python/custom_actions/configs/install_packages_repository_file.json" + with open(config_file, "r") as f: + command_json = json.load(f) + + command_json['repositoryFile']['repoVersion'] = '2.2.0.1-990' + + self.executeScript("scripts/install_packages.py", + classname="InstallPackages", + command="actionexecute", + config_dict=command_json, + target=RMFTestCase.TARGET_CUSTOM_ACTIONS, + os_type=('Suse', '11', 'Final'), + ) + self.assertTrue(put_structured_out_mock.called) + self.assertEquals(put_structured_out_mock.call_args[0][0], + {'package_installation_result': 'SUCCESS', + 'installed_repository_version': VERSION_STUB, + 'stack_id': 'HDP-2.2', + 'actual_version': VERSION_STUB}) + self.assertResourceCalled('Repository', 'HDP-UTILS-1.1.0.20-repo-4', + base_url=u'http://repo1/HDP-UTILS/centos5/2.x/updates/2.2.0.0', + action=['create'], + components=[u'HDP-UTILS', 'main'], + repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', + repo_file_name=u'ambari-hdp-4', + mirror_list=None, + append_to_file=False, + ) + self.assertResourceCalled('Repository', 'HDP-2.2-repo-4', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', + action=['create'], + components=[u'HDP', 'main'], + repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', + repo_file_name=u'ambari-hdp-4', + mirror_list=None, + append_to_file=True, + ) + self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_990', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_990', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_990-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertNoMoreResources() + + @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos") @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") @patch("resource_management.libraries.script.Script.put_structured_out")