Repository: ambari Updated Branches: refs/heads/trunk f2b8164e3 -> 95798ad96
AMBARI-15447. Use Version Definition value for package-version when installing (ncole) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/95798ad9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/95798ad9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/95798ad9 Branch: refs/heads/trunk Commit: 95798ad96a19902dabec855c65a96bcfab63400d Parents: f2b8164 Author: Nate Cole <nc...@hortonworks.com> Authored: Wed Mar 16 17:03:38 2016 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Thu Mar 17 15:07:14 2016 -0400 ---------------------------------------------------------------------- .../libraries/script/script.py | 20 +- .../ambari/server/agent/ExecutionCommand.java | 10 +- .../AmbariManagementControllerImpl.java | 125 ++++++----- .../ClusterStackVersionResourceProvider.java | 25 ++- .../HostStackVersionResourceProvider.java | 22 +- .../orm/entities/RepositoryVersionEntity.java | 12 +- .../ambari/server/state/repository/Release.java | 6 + ...ClusterStackVersionResourceProviderTest.java | 213 ++++++++++++++++++- .../custom_actions/TestInstallPackages.py | 71 +++++++ .../src/test/resources/hbase_version_test.xml | 1 + 10 files changed, 432 insertions(+), 73 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-common/src/main/python/resource_management/libraries/script/script.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py index a8098a0..1b0116e 100644 --- a/ambari-common/src/main/python/resource_management/libraries/script/script.py +++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py @@ -274,11 +274,27 @@ class Script(object): return Script.stack_version_from_distro_select def format_package_name(self, name): + from resource_management.libraries.functions.default import default """ - This function replaces ${stack_version} placeholder into actual version. + This function replaces ${stack_version} placeholder into actual version. If the package + version is passed from the server, use that as an absolute truth. """ + + # two different command types put things in different objects. WHY. + # package_version is the form W_X_Y_Z_nnnn + package_version = default("roleParams/package_version", None) + if not package_version: + package_version = default("hostLevelParams/package_version", None) + package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_' - stack_version_package_formatted = self.get_stack_version_before_packages_installed().replace('.', package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name + + if package_version: + stack_version_package_formatted = package_version + if OSCheck.is_ubuntu_family(): + stack_version_package_formatted = package_version.replace('_', package_delimiter) + else: + stack_version_package_formatted = self.get_stack_version_before_packages_installed().replace('.', package_delimiter).replace('-', package_delimiter) if STACK_VERSION_PLACEHOLDER in name else name + package_name = name.replace(STACK_VERSION_PLACEHOLDER, stack_version_package_formatted) return package_name http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 402a338..9ea541e 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -100,8 +100,8 @@ public class ExecutionCommand extends AgentCommand { private String serviceName; @SerializedName("serviceType") - private String serviceType; - + private String serviceType; + @SerializedName("componentName") private String componentName; @@ -284,7 +284,7 @@ public class ExecutionCommand extends AgentCommand { public void setServiceName(String serviceName) { this.serviceName = serviceName; } - + public String getServiceType() { return serviceType; } @@ -383,6 +383,10 @@ public class ExecutionCommand extends AgentCommand { String SERVICE_CHECK = "SERVICE_CHECK"; // TODO: is it standard command? maybe add it to RoleCommand enum? String CUSTOM_COMMAND = "custom_command"; + /** + * The key indicating that the package_version string is available + */ + String PACKAGE_VERSION = "package_version"; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index e32e2f9..6a12aa3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -18,16 +18,52 @@ package org.apache.ambari.server.controller; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.Multimap; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import com.google.inject.Inject; -import com.google.inject.Injector; -import com.google.inject.Singleton; -import com.google.inject.persist.Transactional; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_DRIVER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_PASSWORD; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MAX_DURATION_OF_RETRIES; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_VERSION; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; +import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.lang.reflect.Type; +import java.net.InetAddress; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; + import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.ClusterNotFoundException; import org.apache.ambari.server.DuplicateResourceException; @@ -120,6 +156,7 @@ import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.StackInfo; import org.apache.ambari.server.state.State; import org.apache.ambari.server.state.configgroup.ConfigGroupFactory; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.scheduler.RequestExecutionFactory; import org.apache.ambari.server.state.stack.WidgetLayout; import org.apache.ambari.server.state.stack.WidgetLayoutInfo; @@ -137,50 +174,16 @@ import org.apache.http.client.utils.URIBuilder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.lang.reflect.Type; -import java.net.InetAddress; -import java.text.MessageFormat; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.EnumMap; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; -import java.util.TreeMap; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Lock; - -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_DRIVER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_PASSWORD; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_URL; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.AMBARI_DB_RCA_USERNAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.CLIENTS_TO_UPDATE_CONFIGS; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_RETRY_ENABLED; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.COMMAND_TIMEOUT; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.DB_DRIVER_FILENAME; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.GROUP_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.NOT_MANAGED_HDFS_PATH_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.HOOKS_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.MAX_DURATION_OF_RETRIES; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.PACKAGE_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.REPO_INFO; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SCRIPT_TYPE; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_PACKAGE_FOLDER; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.SERVICE_REPO_INFO; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.USER_LIST; -import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.VERSION; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import com.google.inject.Inject; +import com.google.inject.Injector; +import com.google.inject.Singleton; +import com.google.inject.persist.Transactional; @Singleton public class AmbariManagementControllerImpl implements AmbariManagementController { @@ -2171,6 +2174,20 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle hostParams.put(REPO_INFO, repoInfo); hostParams.putAll(getRcaParameters()); + RepositoryVersionEntity repoVersion = (null == cluster.getCurrentClusterVersion()) ? null : + cluster.getCurrentClusterVersion().getRepositoryVersion(); + if (null != repoVersion) { + try { + VersionDefinitionXml xml = repoVersion.getRepositoryXml(); + if (null != xml && !StringUtils.isBlank(xml.release.packageVersion)) { + hostParams.put(PACKAGE_VERSION, xml.release.packageVersion); + } + } catch (Exception e) { + throw new AmbariException(String.format("Could not load version xml from repo version %s", + repoVersion.getVersion()), e); + } + } + List<ServiceOsSpecific.Package> packages = getPackagesForServiceHost(serviceInfo, hostParams, osFamily); String packageList = gson.toJson(packages); http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index 07e62b3..6f3c03c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -41,6 +41,7 @@ import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.ActionExecutionContext; @@ -78,6 +79,7 @@ import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.ServiceOsSpecific; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.StringUtils; @@ -423,7 +425,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou for (int i = 0; i < maxTasks && hostIterator.hasNext(); i++) { Host host = hostIterator.next(); if (hostHasVersionableComponents(cluster, serviceNames, ami, stackId, host)) { - ActionExecutionContext actionContext = getHostVersionInstallCommand(desiredRepoVersion, + ActionExecutionContext actionContext = getHostVersionInstallCommand(repoVersionEnt, cluster, managementController, ami, stackId, serviceNames, perOsRepos, stage, host); if (null != actionContext) { try { @@ -488,7 +490,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou return getRequestStatus(req.getRequestStatusResponse()); } - private ActionExecutionContext getHostVersionInstallCommand(final String desiredRepoVersion, + private ActionExecutionContext getHostVersionInstallCommand(RepositoryVersionEntity repoVersion, Cluster cluster, AmbariManagementController managementController, AmbariMetaInfo ami, final StackId stackId, Set<String> repoServices, Map<String, List<RepositoryEntity>> perOsRepos, Stage stage1, Host host) throws SystemException { @@ -498,7 +500,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou if (repoInfo == null) { throw new SystemException(String.format("Repositories for os type %s are " + "not defined. Repo version=%s, stackId=%s", - osFamily, desiredRepoVersion, stackId)); + osFamily, repoVersion.getVersion(), stackId)); } @@ -548,9 +550,22 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou Map<String, String> params = new HashMap<String, String>(); params.put("stack_id", stackId.getStackId()); - params.put("repository_version", desiredRepoVersion); + params.put("repository_version", repoVersion.getVersion()); params.put("base_urls", repoList); - params.put("package_list", packageList); + params.put(KeyNames.PACKAGE_LIST, packageList); + + VersionDefinitionXml xml = null; + try { + xml = repoVersion.getRepositoryXml(); + } catch (Exception e) { + throw new SystemException(String.format("Could not load xml from repo version %s", + repoVersion.getVersion())); + } + + if (null != xml && StringUtils.isNotBlank(xml.release.packageVersion)) { + params.put(KeyNames.PACKAGE_VERSION, xml.release.packageVersion); + } + // add host to this stage RequestResourceFilter filter = new RequestResourceFilter(null, null, http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java index 1cd9c0a..b3f353c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java @@ -28,13 +28,13 @@ import java.util.List; import java.util.Map; import java.util.Set; -import com.google.common.collect.Sets; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.StaticallyInject; import org.apache.ambari.server.actionmanager.ActionManager; import org.apache.ambari.server.actionmanager.RequestFactory; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; +import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.ActionExecutionContext; @@ -64,12 +64,15 @@ import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.state.ServiceOsSpecific; import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.utils.StageUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.Validate; +import com.google.common.collect.Sets; import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Provider; -import org.apache.commons.lang.Validate; /** * Resource provider for host stack versions resources. @@ -391,9 +394,22 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource put("stack_id", stackId.getStackId()); put("repository_version", desiredRepoVersion); put("base_urls", repoList); - put("package_list", packageList); + put(KeyNames.PACKAGE_LIST, packageList); }}; + VersionDefinitionXml xml = null; + try { + xml = repoVersionEnt.getRepositoryXml(); + } catch (Exception e) { + throw new SystemException(String.format("Could not load xml from repo version %s", + repoVersionEnt.getVersion())); + } + + if (null != xml && StringUtils.isNotBlank(xml.release.packageVersion)) { + params.put(KeyNames.PACKAGE_VERSION, xml.release.packageVersion); + } + + // Create custom action RequestResourceFilter filter = new RequestResourceFilter(null, null, Collections.singletonList(hostName)); http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java index 3398709..db393b9 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/orm/entities/RepositoryVersionEntity.java @@ -40,6 +40,7 @@ import javax.persistence.OneToMany; import javax.persistence.OneToOne; import javax.persistence.Table; import javax.persistence.TableGenerator; +import javax.persistence.Transient; import javax.persistence.UniqueConstraint; import org.apache.ambari.server.StaticallyInject; @@ -119,6 +120,9 @@ public class RepositoryVersionEntity { @Column(name="version_xml", insertable = true, updatable = true) private String versionXml; + @Transient + private VersionDefinitionXml versionDefinition = null; + @Column(name="version_url", nullable=true, insertable=true, updatable=true) private String versionUrl; @@ -348,7 +352,7 @@ public class RepositoryVersionEntity { /** * Parse the version XML into its object representation. This causes the XML to be lazy-loaded - * from storage. + * from storage, and will only be parsed once per request. * @return {@code null} if the XSD (from the XML) is not available. * @throws Exception */ @@ -357,7 +361,11 @@ public class RepositoryVersionEntity { return null; } - return VersionDefinitionXml.load(getVersionXml()); + if (null == versionDefinition) { + versionDefinition = VersionDefinitionXml.load(getVersionXml()); + } + + return versionDefinition; } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java index 6bcedf5..f855b05 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/repository/Release.java @@ -75,6 +75,12 @@ public class Release { public String display; /** + * The optional package version + */ + @XmlElement(name="package-version") + public String packageVersion; + + /** * @return the full version */ public String getFullVersion() { http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java index dea83a1..58f00e9 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java @@ -53,6 +53,7 @@ import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; @@ -357,8 +358,6 @@ public class ClusterStackVersionResourceProviderTest { // check that the success factor was populated in the stage Float successFactor = successFactors.get(Role.INSTALL_PACKAGES); Assert.assertEquals(Float.valueOf(0.85f), successFactor); - - } @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) @@ -579,9 +578,214 @@ public class ClusterStackVersionResourceProviderTest { // check that the success factor was populated in the stage Float successFactor = successFactors.get(Role.INSTALL_PACKAGES); Assert.assertEquals(Float.valueOf(0.85f), successFactor); - } + @Test + public void testCreateResourcesWithRepoDefinition() throws Exception { + Resource.Type type = Resource.Type.ClusterStackVersion; + + AmbariManagementController managementController = createMock(AmbariManagementController.class); + Clusters clusters = createNiceMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + StackId stackId = new StackId("HDP", "2.0.1"); + + File f = new File("src/test/resources/hbase_version_test.xml"); + + RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); + repoVersion.setId(1l); + repoVersion.setOperatingSystems(OS_JSON); + repoVersion.setVersionXml(IOUtils.toString(new FileInputStream(f))); + repoVersion.setVersionXsd("version_definition.xsd"); + repoVersion.setType(RepositoryType.STANDARD); + + ambariMetaInfo.getComponent("HDP", "2.1.1", "HBASE", "HBASE_MASTER").setVersionAdvertised(true); + + + Map<String, Host> hostsForCluster = new HashMap<String, Host>(); + int hostCount = 10; + for (int i = 0; i < hostCount; i++) { + String hostname = "host" + i; + Host host = createNiceMock(hostname, Host.class); + expect(host.getHostName()).andReturn(hostname).anyTimes(); + expect(host.getOsFamily()).andReturn("redhat6").anyTimes(); + expect(host.getMaintenanceState(EasyMock.anyLong())).andReturn( + MaintenanceState.OFF).anyTimes(); + expect(host.getAllHostVersions()).andReturn( + Collections.<HostVersionEntity>emptyList()).anyTimes(); + + replay(host); + hostsForCluster.put(hostname, host); + } + + Service hdfsService = createNiceMock(Service.class); + Service hbaseService = createNiceMock(Service.class); + expect(hdfsService.getName()).andReturn("HDFS").anyTimes(); + expect(hbaseService.getName()).andReturn("HBASE").anyTimes(); + + expect(hdfsService.getServiceComponents()).andReturn(new HashMap<String, ServiceComponent>()); + expect(hbaseService.getServiceComponents()).andReturn(new HashMap<String, ServiceComponent>()); + + Map<String, Service> serviceMap = new HashMap<>(); + serviceMap.put("HDFS", hdfsService); + serviceMap.put("HBASE", hbaseService); + + final ServiceComponentHost schDatanode = createMock(ServiceComponentHost.class); + expect(schDatanode.getServiceName()).andReturn("HDFS").anyTimes(); + expect(schDatanode.getServiceComponentName()).andReturn("DATANODE").anyTimes(); + + final ServiceComponentHost schNamenode = createMock(ServiceComponentHost.class); + expect(schNamenode.getServiceName()).andReturn("HDFS").anyTimes(); + expect(schNamenode.getServiceComponentName()).andReturn("NAMENODE").anyTimes(); + + final ServiceComponentHost schHBM = createMock(ServiceComponentHost.class); + expect(schHBM.getServiceName()).andReturn("HBASE").anyTimes(); + expect(schHBM.getServiceComponentName()).andReturn("HBASE_MASTER").anyTimes(); + + // First host contains versionable components + final List<ServiceComponentHost> schsH1 = Arrays.asList(schDatanode, schNamenode); + + // Second host contains versionable components + final List<ServiceComponentHost> schsH2 = Arrays.asList(schDatanode); + + // Third host only has hbase + final List<ServiceComponentHost> schsH3 = Arrays.asList(schHBM); + + ServiceOsSpecific.Package hdfsPackage = new ServiceOsSpecific.Package(); + hdfsPackage.setName("hdfs"); + + List<ServiceOsSpecific.Package> packages = Collections.singletonList(hdfsPackage); + + ActionManager actionManager = createNiceMock(ActionManager.class); + + RequestStatusResponse response = createNiceMock(RequestStatusResponse.class); + ResourceProviderFactory resourceProviderFactory = createNiceMock(ResourceProviderFactory.class); + ResourceProvider csvResourceProvider = createNiceMock(ClusterStackVersionResourceProvider.class); + + AbstractControllerResourceProvider.init(resourceProviderFactory); + + Map<String, Map<String, String>> hostConfigTags = new HashMap<String, Map<String, String>>(); + expect(configHelper.getEffectiveDesiredTags(anyObject(ClusterImpl.class), anyObject(String.class))).andReturn(hostConfigTags); + + expect(managementController.getClusters()).andReturn(clusters).anyTimes(); + expect(managementController.getAmbariMetaInfo()).andReturn(ambariMetaInfo).anyTimes(); + expect(managementController.getAuthName()).andReturn("admin").anyTimes(); + expect(managementController.getActionManager()).andReturn(actionManager).anyTimes(); + expect(managementController.getJdkResourceUrl()).andReturn("/JdkResourceUrl").anyTimes(); + expect(managementController.getPackagesForServiceHost(anyObject(ServiceInfo.class), + (Map<String, String>) anyObject(List.class), anyObject(String.class))). + andReturn(packages).anyTimes(); // only one host has the versionable component + + expect(resourceProviderFactory.getHostResourceProvider(anyObject(Set.class), anyObject(Map.class), + eq(managementController))).andReturn(csvResourceProvider).anyTimes(); + + expect(clusters.getCluster(anyObject(String.class))).andReturn(cluster); + expect(clusters.getHostsForCluster(anyObject(String.class))).andReturn( + hostsForCluster).anyTimes(); + + String clusterName = "Cluster100"; + expect(cluster.getClusterId()).andReturn(1L).anyTimes(); + expect(cluster.getHosts()).andReturn(hostsForCluster.values()).atLeastOnce(); + expect(cluster.getServices()).andReturn(serviceMap).anyTimes(); + expect(cluster.getCurrentStackVersion()).andReturn(stackId); + expect(cluster.getServiceComponentHosts(anyObject(String.class))).andAnswer(new IAnswer<List<ServiceComponentHost>>() { + @Override + public List<ServiceComponentHost> answer() throws Throwable { + String hostname = (String) EasyMock.getCurrentArguments()[0]; + if (hostname.equals("host2")) { + return schsH2; + } else if (hostname.equals("host3")) { + return schsH3; + } else { + return schsH1; + } + } + }).anyTimes(); + +// ExecutionCommand executionCommand = createNiceMock(ExecutionCommand.class); + ExecutionCommand executionCommand = new ExecutionCommand(); + ExecutionCommandWrapper executionCommandWrapper = createNiceMock(ExecutionCommandWrapper.class); + +// expect(executionCommand.getHostLevelParams()).andReturn(new HashMap<String, String>()).atLeastOnce(); + expect(executionCommandWrapper.getExecutionCommand()).andReturn(executionCommand).anyTimes(); + + Stage stage = createNiceMock(Stage.class); + expect(stage.getExecutionCommandWrapper(anyObject(String.class), anyObject(String.class))). + andReturn(executionCommandWrapper).anyTimes(); + + Map<Role, Float> successFactors = new HashMap<>(); + expect(stage.getSuccessFactors()).andReturn(successFactors).atLeastOnce(); + + // Check that we create proper stage count + expect(stageFactory.createNew(anyLong(), anyObject(String.class), + anyObject(String.class), anyLong(), + anyObject(String.class), anyObject(String.class), anyObject(String.class), + anyObject(String.class))).andReturn(stage). + times((int) Math.ceil(hostCount / MAX_TASKS_PER_STAGE)); + + expect( + repositoryVersionDAOMock.findByStackAndVersion( + anyObject(StackId.class), + anyObject(String.class))).andReturn(repoVersion); + + Capture<org.apache.ambari.server.actionmanager.Request> c = Capture.newInstance(); + Capture<ExecuteActionRequest> ear = Capture.newInstance(); + + actionManager.sendActions(capture(c), capture(ear)); + expectLastCall().atLeastOnce(); + expect(actionManager.getRequestTasks(anyLong())).andReturn(Collections.<HostRoleCommand>emptyList()).anyTimes(); + + ClusterEntity clusterEntity = new ClusterEntity(); + clusterEntity.setClusterId(1l); + clusterEntity.setClusterName(clusterName); + ClusterVersionEntity cve = new ClusterVersionEntity(clusterEntity, + repoVersion, RepositoryVersionState.INSTALL_FAILED, 0, ""); + expect(clusterVersionDAO.findByClusterAndStackAndVersion(anyObject(String.class), + anyObject(StackId.class), anyObject(String.class))).andReturn(cve); + + TopologyManager topologyManager = injector.getInstance(TopologyManager.class); + StageUtils.setTopologyManager(topologyManager); + + // replay + replay(managementController, response, clusters, hdfsService, hbaseService, resourceProviderFactory, csvResourceProvider, + cluster, repositoryVersionDAOMock, configHelper, schDatanode, schNamenode, schHBM, actionManager, + executionCommandWrapper,stage, stageFactory, clusterVersionDAO); + + ResourceProvider provider = AbstractControllerResourceProvider.getResourceProvider( + type, + PropertyHelper.getPropertyIds(type), + PropertyHelper.getKeyPropertyIds(type), + managementController); + + injector.injectMembers(provider); + + // add the property map to a set for the request. add more maps for multiple creates + Set<Map<String, Object>> propertySet = new LinkedHashSet<Map<String, Object>>(); + + Map<String, Object> properties = new LinkedHashMap<String, Object>(); + + // add properties to the request map + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_CLUSTER_NAME_PROPERTY_ID, "Cluster100"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_REPOSITORY_VERSION_PROPERTY_ID, "2.2.0.1-885"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_STACK_PROPERTY_ID, "HDP"); + properties.put(ClusterStackVersionResourceProvider.CLUSTER_STACK_VERSION_VERSION_PROPERTY_ID, "2.1.1"); + + propertySet.add(properties); + + // create the request + Request request = PropertyHelper.getCreateRequest(propertySet, null); + + RequestStatus status = provider.createResources(request); + Assert.assertNotNull(status); + + // verify + verify(managementController, response, clusters, stageFactory, stage); + + // check that the success factor was populated in the stage + Float successFactor = successFactors.get(Role.INSTALL_PACKAGES); + Assert.assertEquals(Float.valueOf(0.85f), successFactor); + + Assert.assertTrue(executionCommand.getRoleParams().containsKey(KeyNames.PACKAGE_VERSION)); + } /** * Tests manual finalization scenario @@ -885,7 +1089,8 @@ public class ClusterStackVersionResourceProviderTest { bind(ClusterVersionDAO.class).toInstance(clusterVersionDAO); bind(HostVersionDAO.class).toInstance(hostVersionDAO); bind(HostComponentStateDAO.class).toInstance(hostComponentStateDAO); - } } + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/test/python/custom_actions/TestInstallPackages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py index f022c80..ad4206b 100644 --- a/ambari-server/src/test/python/custom_actions/TestInstallPackages.py +++ b/ambari-server/src/test/python/custom_actions/TestInstallPackages.py @@ -377,6 +377,77 @@ class TestInstallPackages(RMFTestCase): self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) self.assertNoMoreResources() + @patch("ambari_commons.os_check.OSCheck.is_suse_family") + @patch("resource_management.core.resources.packaging.Package") + @patch("resource_management.libraries.script.Script.put_structured_out") + @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") + @patch("resource_management.libraries.functions.stack_select.get_stack_versions") + @patch("resource_management.libraries.functions.repo_version_history.read_actual_version_from_history_file") + @patch("resource_management.libraries.functions.repo_version_history.write_actual_version_to_history_file") + def test_format_package_name_specific(self, write_actual_version_to_history_file_mock, + read_actual_version_from_history_file_mock, + stack_versions_mock, + allInstalledPackages_mock, put_structured_out_mock, + package_mock, is_suse_family_mock): + Script.stack_version_from_distro_select = VERSION_STUB + stack_versions_mock.side_effect = [ + [], # before installation attempt + [VERSION_STUB] + ] + read_actual_version_from_history_file_mock.return_value = VERSION_STUB + allInstalledPackages_mock = MagicMock(side_effect = TestInstallPackages._add_packages) + is_suse_family_mock.return_value = True + + + config_file = self.get_src_folder() + "/test/python/custom_actions/configs/install_packages_config.json" + with open(config_file, "r") as f: + command_json = json.load(f) + + command_json['roleParams']['package_version'] = "2_2_0_1_889" + + + self.executeScript("scripts/install_packages.py", + classname="InstallPackages", + command="actionexecute", + config_dict=command_json, + target=RMFTestCase.TARGET_CUSTOM_ACTIONS, + os_type=('Suse', '11', 'Final'), + ) + self.assertTrue(put_structured_out_mock.called) + self.assertEquals(put_structured_out_mock.call_args[0][0], + {'package_installation_result': 'SUCCESS', + 'installed_repository_version': VERSION_STUB, + 'stack_id': 'HDP-2.2', + 'actual_version': VERSION_STUB, + 'ambari_repositories': []}) + self.assertResourceCalled('Repository', 'HDP-UTILS-2.2.0.1-885', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', + action=['create'], + components=[u'HDP-UTILS', 'main'], + repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', + repo_file_name=u'HDP-2.2.0.1-885', + mirror_list=None, + append_to_file=False, + ) + self.assertResourceCalled('Repository', 'HDP-2.2.0.1-885', + base_url=u'http://repo1/HDP/centos5/2.x/updates/2.2.0.0', + action=['create'], + components=[u'HDP', 'main'], + repo_template=u'[{{repo_id}}]\nname={{repo_id}}\n{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}\n\npath=/\nenabled=1\ngpgcheck=0', + repo_file_name=u'HDP-2.2.0.1-885', + mirror_list=None, + append_to_file=True, + ) + self.assertResourceCalled('Package', 'hdp-select', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_889', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'snappy-devel', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'lzo', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadooplzo_2_2_0_1_889', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'hadoop_2_2_0_1_889-libhdfs', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertResourceCalled('Package', 'ambari-log4j', action=["upgrade"], retry_count=5, retry_on_repo_unavailability=False) + self.assertNoMoreResources() + @patch("resource_management.libraries.functions.list_ambari_managed_repos.list_ambari_managed_repos") @patch("resource_management.libraries.functions.packages_analyzer.allInstalledPackages") http://git-wip-us.apache.org/repos/asf/ambari/blob/95798ad9/ambari-server/src/test/resources/hbase_version_test.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/resources/hbase_version_test.xml b/ambari-server/src/test/resources/hbase_version_test.xml index 9df07ed..183da8c 100644 --- a/ambari-server/src/test/resources/hbase_version_test.xml +++ b/ambari-server/src/test/resources/hbase_version_test.xml @@ -26,6 +26,7 @@ <build>3396</build> <compatible-with>2.3.2.[0-9]</compatible-with> <release-notes>http://docs.hortonworks.com/HDPDocuments/HDP2/HDP-2.3.4/</release-notes> + <package-version>2_3_4_0_3396</package-version> </release> <manifest>
