AMBARI-20871. Unify repository file creation (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b6866249 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b6866249 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b6866249 Branch: refs/heads/branch-feature-AMBARI-12556 Commit: b68662490a64277f4d5eb7ae18b54850678deff2 Parents: 13d806b Author: Nate Cole <nc...@hortonworks.com> Authored: Thu Apr 27 10:15:29 2017 -0400 Committer: Nate Cole <nc...@hortonworks.com> Committed: Thu Apr 27 16:11:15 2017 -0400 ---------------------------------------------------------------------- .../libraries/functions/repository_util.py | 127 ++ .../ambari/server/agent/CommandRepository.java | 170 +++ .../ambari/server/agent/ExecutionCommand.java | 32 +- .../commands/StackAdvisorCommand.java | 6 +- .../DistributeRepositoriesStructuredOutput.java | 12 - .../controller/ActionExecutionContext.java | 35 + .../controller/AmbariActionExecutionHelper.java | 2 + .../AmbariCustomCommandExecutionHelper.java | 222 ++- .../AmbariManagementControllerImpl.java | 4 +- .../ClusterStackVersionResourceProvider.java | 105 +- .../HostStackVersionResourceProvider.java | 70 +- .../stack/upgrade/RepositoryVersionHelper.java | 130 +- .../custom_actions/scripts/install_packages.py | 63 +- .../hooks/before-INSTALL/scripts/params.py | 2 + .../scripts/repo_initialization.py | 22 +- ...ClusterStackVersionResourceProviderTest.java | 19 +- .../HostStackVersionResourceProviderTest.java | 3 +- .../upgrades/UpgradeActionTest.java | 15 +- .../UpdateActiveRepoVersionOnStartupTest.java | 25 +- .../upgrade/RepositoryVersionHelperTest.java | 27 +- .../custom_actions/TestInstallPackages.py | 118 +- .../install_packages_repository_file.json | 112 ++ .../stacks/2.0.6/configs/repository_file.json | 1275 ++++++++++++++++++ .../hooks/before-INSTALL/test_before_install.py | 30 + 24 files changed, 2293 insertions(+), 333 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py ---------------------------------------------------------------------- diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py new file mode 100644 index 0000000..c9920e2 --- /dev/null +++ b/ambari-common/src/main/python/resource_management/libraries/functions/repository_util.py @@ -0,0 +1,127 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.exceptions import Fail +from resource_management.core.logger import Logger +from resource_management.libraries.resources.repository import Repository +import ambari_simplejson as json + + +__all__ = ["create_repo_files", "CommandRepository"] + +# components_lits = repoName + postfix +UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"] + + +def create_repo_files(template, command_repository): + """ + Creates repositories in a consistent manner for all types + :param command_repository: a CommandRepository instance + :return: + """ + + if command_repository.version_id is None: + raise Fail("The command repository was not parsed correctly") + + if 0 == len(command_repository.repositories): + raise Fail("Cannot create repository files when no repositories are defined") + + # add the stack name to the file name just to make it a little easier to debug + # version_id is the primary id of the repo_version table in the database + file_name = "ambari-{0}-{1}".format(command_repository.stack_name.lower(), + command_repository.version_id) + + append_to_file = False # initialize to False to create the file anew. + + for repository in command_repository.repositories: + + if repository.repo_id is None: + raise Fail("Repository with url {0} has no id".format(repository.base_url)) + + Repository(repository.repo_id, + action = "create", + base_url = repository.base_url, + mirror_list = repository.mirrors_list, + repo_file_name = file_name, + repo_template = template, + components = repository.ubuntu_components, + append_to_file = append_to_file) + append_to_file = True + + +def _find_value(dictionary, key): + """ + Helper to find a value in a dictionary + """ + if key not in dictionary: + return None + + return dictionary[key] + + +class CommandRepository(object): + """ + Class that encapsulates the representation of repositories passed in a command. This class + should match the CommandRepository class. + """ + + def __init__(self, jsonvalue): + + if isinstance(jsonvalue, dict): + json_dict = jsonvalue + elif isinstance(jsonvalue, basestring): + json_dict = json.loads(jsonvalue) + + if json_dict is None: + raise Fail("Cannot deserialize command repository {0}".format(str(jsonvalue))) + + # version_id is the primary id of the repo_version table in the database + self.version_id = _find_value(json_dict, 'repoVersionId') + self.stack_name = _find_value(json_dict, 'stackName') + self.repositories = [] + + repos_def = _find_value(json_dict, 'repositories') + if repos_def is not None: + if not isinstance(repos_def, list): + repos_def = [repos_def] + + for repo_def in repos_def: + self.repositories.append(_CommandRepositoryEntry(repo_def)) + + +class _CommandRepositoryEntry(object): + """ + Class that represents the entries of a CommandRepository. This isn't meant to be instantiated + outside a CommandRepository + """ + def __init__(self, json_dict): + self.repo_id = _find_value(json_dict, 'repoId') # this is the id within the repo file, not an Ambari artifact + self.repo_name = _find_value(json_dict, 'repoName') + self.base_url = _find_value(json_dict, 'baseUrl') + self.mirrors_list = _find_value(json_dict, 'mirrorsList') + + # if repoName is changed on the java side, this will fail for ubuntu since we rely on the + # name being the same as how the repository was built + self.ubuntu_components = [self.repo_name] + UBUNTU_REPO_COMPONENTS_POSTFIX + + + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java new file mode 100644 index 0000000..849d6fb --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/CommandRepository.java @@ -0,0 +1,170 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.agent; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.ambari.server.orm.entities.RepositoryEntity; +import org.apache.ambari.server.state.RepositoryInfo; + +import com.google.gson.annotations.SerializedName; + +/** + * Wraps the information required to create repositories from a command. This was added + * as a top level command object. + */ +public class CommandRepository { + + @SerializedName("repositories") + private List<Repository> m_repositories = new ArrayList<>(); + + @SerializedName("repoVersion") + private String m_repoVersion; + + @SerializedName("repoVersionId") + private long m_repoVersionId; + + @SerializedName("stackName") + private String m_stackName; + + /** + * @param version the repo version + */ + public void setRepositoryVersion(String version) { + m_repoVersion = version; + } + + /** + * @param id the repository id + */ + public void setRepositoryVersionId(long id) { + m_repoVersionId = id; + } + + /** + * @param name the stack name + */ + public void setStackName(String name) { + m_stackName = name; + } + + /** + * @param repositories the repositories if sourced from the stack instead of the repo_version. + */ + public void setRepositories(Collection<RepositoryInfo> repositories) { + m_repositories = new ArrayList<>(); + + for (RepositoryInfo info : repositories) { + m_repositories.add(new Repository(info)); + } + } + + /** + * @param osType the OS type for the repositories + * @param repositories the repository entities that should be processed into a file + */ + public void setRepositories(String osType, Collection<RepositoryEntity> repositories) { + m_repositories = new ArrayList<>(); + + for (RepositoryEntity entity : repositories) { + m_repositories.add(new Repository(osType, entity)); + } + } + + /** + * @return the repositories that the command should process into a file. + */ + public Collection<Repository> getRepositories() { + return m_repositories; + } + + /** + * Sets a uniqueness on the repo ids. + * + * @param suffix the repo id suffix + */ + public void setUniqueSuffix(String suffix) { + for (Repository repo : m_repositories) { + repo.m_repoId = repo.m_repoId + suffix; + } + } + + /** + * Minimal information required to generate repo files on the agent. These are copies + * of the repository objects from repo versions that can be changed for URL overrides, etc. + */ + public static class Repository { + + @SerializedName("baseUrl") + private String m_baseUrl; + + @SerializedName("repoId") + private String m_repoId; + + /** + * The name should not change. Ubuntu requires that it match exactly as the repo was built. + */ + @SerializedName("repoName") + private final String m_repoName; + + @SerializedName("mirrorsList") + private String m_mirrorsList; + + private transient String m_osType; + + private Repository(RepositoryInfo info) { + m_baseUrl = info.getBaseUrl(); + m_osType = info.getOsType(); + m_repoId = info.getRepoId(); + m_repoName = info.getRepoName(); + m_mirrorsList = info.getMirrorsList(); + } + + private Repository(String osType, RepositoryEntity entity) { + m_baseUrl = entity.getBaseUrl(); + m_repoId = entity.getRepositoryId(); + m_repoName = entity.getName(); + m_mirrorsList = entity.getMirrorsList(); + m_osType = osType; + } + + public void setBaseUrl(String url) { + m_baseUrl = url; + } + + public String getOsType() { + return m_osType; + } + + public String getRepoId() { + return m_repoId; + } + + public String getRepoName() { + return m_repoName; + } + + + public String getBaseUrl() { + return m_baseUrl; + } + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java index 8c726a0..bd62cbb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java @@ -24,6 +24,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.state.ServiceInfo; import org.apache.ambari.server.utils.StageUtils; @@ -76,8 +78,7 @@ public class ExecutionCommand extends AgentCommand { private RoleCommand roleCommand; @SerializedName("clusterHostInfo") - private Map<String, Set<String>> clusterHostInfo = - new HashMap<>(); + private Map<String, Set<String>> clusterHostInfo = new HashMap<>(); @SerializedName("configurations") private Map<String, Map<String, String>> configurations; @@ -144,6 +145,14 @@ public class ExecutionCommand extends AgentCommand { @SerializedName("configuration_credentials") private Map<String, Map<String, String>> configurationCredentials; + + /** + * Provides information regarding the content of repositories. This structure replaces + * the deprecated use of {@link KeyNames#REPO_INFO} + */ + @SerializedName("repositoryFile") + private CommandRepository commandRepository; + public void setConfigurationCredentials(Map<String, Map<String, String>> configurationCredentials) { this.configurationCredentials = configurationCredentials; } @@ -395,6 +404,20 @@ public class ExecutionCommand extends AgentCommand { } /** + * @return the repository file that is to be written. + */ + public CommandRepository getRepositoryFile() { + return commandRepository; + } + + /** + * @param repository the command repository instance. + */ + public void setRepositoryFile(CommandRepository repository) { + commandRepository = repository; + } + + /** * Contains key name strings. These strings are used inside maps * incapsulated inside command. */ @@ -420,6 +443,11 @@ public class ExecutionCommand extends AgentCommand { String ORACLE_JDBC_URL = "oracle_jdbc_url"; String DB_DRIVER_FILENAME = "db_driver_filename"; String CLIENTS_TO_UPDATE_CONFIGS = "clientsToUpdateConfigs"; + /** + * Keep for backward compatibility. + */ + @Deprecated + @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) String REPO_INFO = "repo_info"; String DB_NAME = "db_name"; String GLOBAL = "global"; http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java index 74a2cf2..a291a88 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/services/stackadvisor/commands/StackAdvisorCommand.java @@ -348,13 +348,13 @@ public abstract class StackAdvisorCommand<T extends StackAdvisorResponse> extend return file.isDirectory() && !FileUtils.isFileNewer(file, cutoffDate); } }); - + if(oldDirectories.length > 0) { LOG.info(String.format("Deleting old directories %s from %s", StringUtils.join(oldDirectories, ", "), recommendationsDir)); } - + for(String oldDirectory:oldDirectories) { - FileUtils.deleteDirectory(new File(recommendationsDir, oldDirectory)); + FileUtils.deleteQuietly(new File(recommendationsDir, oldDirectory)); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java index 662b856..7c961b5 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/DistributeRepositoriesStructuredOutput.java @@ -19,8 +19,6 @@ package org.apache.ambari.server.bootstrap; -import java.util.List; - import com.google.gson.annotations.SerializedName; /** @@ -36,12 +34,6 @@ public class DistributeRepositoriesStructuredOutput { private String installedRepositoryVersion; /** - * All Ambari-managed repositories that are installed side by side on host - */ - @SerializedName("ambari_repositories") - private List<String> ambariRepositories; - - /** * Either SUCCESS or FAIL */ @SerializedName("package_installation_result") @@ -64,10 +56,6 @@ public class DistributeRepositoriesStructuredOutput { return installedRepositoryVersion; } - public List<String> getAmbariRepositories() { - return ambariRepositories; - } - public String getPackageInstallationResult() { return packageInstallationResult; } http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java index 3681eda..5566fe2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ActionExecutionContext.java @@ -19,10 +19,12 @@ package org.apache.ambari.server.controller; +import java.util.ArrayList; import java.util.List; import java.util.Map; import org.apache.ambari.server.actionmanager.TargetHostType; +import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.controller.internal.RequestOperationLevel; import org.apache.ambari.server.controller.internal.RequestResourceFilter; @@ -42,6 +44,8 @@ public class ActionExecutionContext { private boolean hostsInMaintenanceModeExcluded = true; private boolean allowRetry = false; + private List<ExecutionCommandVisitor> m_visitors = new ArrayList<>(); + /** * {@code true} if slave/client component failures should be automatically * skipped. This will only automatically skip the failure if the task is @@ -168,6 +172,16 @@ public class ActionExecutionContext { this.autoSkipFailures = autoSkipFailures; } + /** + * Adds a command visitor that will be invoked after a command is created. Provides access + * to the command. + * + * @param visitor the visitor + */ + public void addVisitor(ExecutionCommandVisitor visitor) { + m_visitors.add(visitor); + } + @Override public String toString() { return "ActionExecutionContext{" + @@ -207,4 +221,25 @@ public class ActionExecutionContext { hostsInMaintenanceModeExcluded = excluded; } + /** + * Called as a way to post-process the command after it has been created and various objects + * have been set. + * + * @param command the command + */ + public void visitAll(ExecutionCommand command) { + for (ExecutionCommandVisitor visitor : m_visitors) { + visitor.visit(command); + } + } + + /** + * Interface that allows a final attempt to setting values on an {@link ExecutionCommand} + * @author ncole + * + */ + public static interface ExecutionCommandVisitor { + public void visit(ExecutionCommand command); + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java index d556b60..835d607 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java @@ -487,6 +487,8 @@ public class AmbariActionExecutionHelper { execCmd.getLocalComponents().add(sch.getServiceComponentName()); } } + + actionContext.visitAll(execCmd); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java index ab8b659..d6905fb 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java @@ -58,6 +58,8 @@ import java.util.Random; import java.util.Set; import java.util.TreeMap; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; @@ -65,6 +67,7 @@ import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.agent.AgentCommand.AgentCommandType; +import org.apache.ambari.server.agent.CommandRepository; import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; @@ -108,6 +111,7 @@ import org.apache.commons.lang.math.NumberUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.base.Function; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -500,6 +504,8 @@ public class AmbariCustomCommandExecutionHelper { execCmd.setCommandParams(commandParams); execCmd.setRoleParams(roleParams); + execCmd.setRepositoryFile(getCommandRepository(cluster, host)); + // perform any server side command related logic - eg - set desired states on restart applyCustomCommandBackendLogic(cluster, serviceName, componentName, commandName, hostName); } @@ -1169,54 +1175,186 @@ public class AmbariCustomCommandExecutionHelper { * * @return the repo info * + * @deprecated use {@link #getCommandRepository(Cluster, Host)} instead. * @throws AmbariException if the repository information can not be obtained */ + @Deprecated public String getRepoInfo(Cluster cluster, Host host) throws AmbariException { - return getRepoInfo(cluster, host.getOsType(), host.getOsFamily(), host.getHostName()); + Function<List<RepositoryInfo>, JsonArray> function = new Function<List<RepositoryInfo>, JsonArray>() { + @Override + public JsonArray apply(List<RepositoryInfo> input) { + return null == input ? null : (JsonArray) gson.toJsonTree(input); + } + }; + + final JsonArray gsonList = getBaseUrls(cluster, host, function); + + if (null == gsonList) { + return ""; + } + + BaseUrlUpdater<JsonArray> updater = new BaseUrlUpdater<JsonArray>(gsonList) { + @Override + public JsonArray apply(final RepositoryVersionEntity rve) { + + JsonArray result = new JsonArray(); + + for (JsonElement e : gsonList) { + JsonObject obj = e.getAsJsonObject(); + + String repoId = obj.has("repoId") ? obj.get("repoId").getAsString() : null; + String repoName = obj.has("repoName") ? obj.get("repoName").getAsString() : null; + String baseUrl = obj.has("baseUrl") ? obj.get("baseUrl").getAsString() : null; + String osType = obj.has("osType") ? obj.get("osType").getAsString() : null; + + if (null == repoId || null == baseUrl || null == osType || null == repoName) { + continue; + } + + for (OperatingSystemEntity ose : rve.getOperatingSystems()) { + if (ose.getOsType().equals(osType) && ose.isAmbariManagedRepos()) { + for (RepositoryEntity re : ose.getRepositories()) { + if (re.getName().equals(repoName) && + re.getRepositoryId().equals(repoId) && + !re.getBaseUrl().equals(baseUrl)) { + obj.addProperty("baseUrl", re.getBaseUrl()); + } + } + result.add(e); + } + } + } + + return result; + } + }; + + return updateBaseUrls(cluster, updater).toString(); + } + + /** + * Builds repository information for inclusion in a command. This replaces escaping json on + * a command. + * + * @param cluster the cluster + * @param host the host + * @return the command repository + * @throws AmbariException + */ + @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) + public CommandRepository getCommandRepository(final Cluster cluster, Host host) throws AmbariException { + + Function<List<RepositoryInfo>, List<RepositoryInfo>> function = new Function<List<RepositoryInfo>, List<RepositoryInfo>>() { + @Override + public List<RepositoryInfo> apply(List<RepositoryInfo> input) { + // !!! just return what is given + return input; + } + }; + + final List<RepositoryInfo> repoInfos = getBaseUrls(cluster, host, function); + + if (null == repoInfos) { + return null; + } + + final CommandRepository command = new CommandRepository(); + StackId stackId = cluster.getDesiredStackVersion(); + command.setRepositories(repoInfos); + command.setStackName(stackId.getStackName()); + + final BaseUrlUpdater<Void> updater = new BaseUrlUpdater<Void>(null) { + @Override + public Void apply(RepositoryVersionEntity rve) { + + command.setRepositoryVersionId(rve.getId()); + command.setRepositoryVersion(rve.getVersion()); + command.setStackName(rve.getStackName()); + command.setUniqueSuffix(String.format("-repo-%s", rve.getId())); + + for (CommandRepository.Repository commandRepo : command.getRepositories()) { + String osType = commandRepo.getOsType(); + String repoId = commandRepo.getRepoId(); + String repoName = commandRepo.getRepoName(); + String baseUrl = commandRepo.getBaseUrl(); + + for (OperatingSystemEntity ose : rve.getOperatingSystems()) { + if (ose.getOsType().equals(osType) && ose.isAmbariManagedRepos()) { + for (RepositoryEntity re : ose.getRepositories()) { + if (re.getName().equals(repoName) && + re.getRepositoryId().equals(repoId) && + !re.getBaseUrl().equals(baseUrl)) { + commandRepo.setBaseUrl(re.getBaseUrl()); + } + } + } + } + } + + return null; + } + }; + + updateBaseUrls(cluster, updater); + + return command; } - public String getRepoInfo(Cluster cluster, String hostOSType, String hostOSFamily, String hostName) throws AmbariException { + /** + * Executed by two different representations of repos. When we are comfortable with the new + * implemenation, this may be removed and called inline in {@link #getCommandRepository(Cluster, Host)} + * + * @param cluster the cluster to isolate the stack + * @param host used to resolve the family for the repositories + * @param function function that will transform the supplied repositories for specific use. + * @return <T> the type as defined by the supplied {@code function}. + * @throws AmbariException + */ + @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES) + private <T> T getBaseUrls(Cluster cluster, Host host, Function<List<RepositoryInfo>, T> function) throws AmbariException { + + String hostOsType = host.getOsType(); + String hostOsFamily = host.getOsFamily(); + String hostName = host.getHostName(); StackId stackId = cluster.getDesiredStackVersion(); Map<String, List<RepositoryInfo>> repos = ambariMetaInfo.getRepository( stackId.getStackName(), stackId.getStackVersion()); - String family = os_family.find(hostOSType); + String family = os_family.find(hostOsType); if (null == family) { - family = hostOSFamily; + family = hostOsFamily; } - JsonElement gsonList = null; + final List<RepositoryInfo> repoInfos; // !!! check for the most specific first - if (repos.containsKey(hostOSType)) { - gsonList = gson.toJsonTree(repos.get(hostOSType)); + if (repos.containsKey(hostOsType)) { + repoInfos = repos.get(hostOsType); } else if (null != family && repos.containsKey(family)) { - gsonList = gson.toJsonTree(repos.get(family)); + repoInfos = repos.get(family); } else { + repoInfos = null; LOG.warn("Could not retrieve repo information for host" + ", hostname=" + hostName + ", clusterName=" + cluster.getClusterName() + ", stackInfo=" + stackId.getStackId()); } - if (null != gsonList) { - gsonList = updateBaseUrls(cluster, JsonArray.class.cast(gsonList)); - return gsonList.toString(); - } else { - return ""; - } + // leave it to function implementation to handle null. + return function.apply(repoInfos); } /** * Checks repo URLs against the current version for the cluster and makes * adjustments to the Base URL when the current is different. - * @param cluster the cluster to load the current version - * @param jsonArray the array containing stack repo data + * + * @param <T> the result after appling the repository version, if found. */ - private JsonArray updateBaseUrls(Cluster cluster, JsonArray jsonArray) throws AmbariException { + @Experimental(feature = ExperimentalFeature.PATCH_UPGRADES) + private <T> T updateBaseUrls(Cluster cluster, BaseUrlUpdater<T> function) throws AmbariException { ClusterVersionEntity cve = cluster.getCurrentClusterVersion(); if (null == cve) { @@ -1235,40 +1373,12 @@ public class AmbariCustomCommandExecutionHelper { if (null == cve || null == cve.getRepositoryVersion()) { LOG.info("Cluster {} has no specific Repository Versions. Using stack-defined values", cluster.getClusterName()); - return jsonArray; + return function.getDefault(); } RepositoryVersionEntity rve = cve.getRepositoryVersion(); - JsonArray result = new JsonArray(); - - for (JsonElement e : jsonArray) { - JsonObject obj = e.getAsJsonObject(); - - String repoId = obj.has("repoId") ? obj.get("repoId").getAsString() : null; - String repoName = obj.has("repoName") ? obj.get("repoName").getAsString() : null; - String baseUrl = obj.has("baseUrl") ? obj.get("baseUrl").getAsString() : null; - String osType = obj.has("osType") ? obj.get("osType").getAsString() : null; - - if (null == repoId || null == baseUrl || null == osType || null == repoName) { - continue; - } - - for (OperatingSystemEntity ose : rve.getOperatingSystems()) { - if (ose.getOsType().equals(osType) && ose.isAmbariManagedRepos()) { - for (RepositoryEntity re : ose.getRepositories()) { - if (re.getName().equals(repoName) && - re.getRepositoryId().equals(repoId) && - !re.getBaseUrl().equals(baseUrl)) { - obj.addProperty("baseUrl", re.getBaseUrl()); - } - } - result.add(e); - } - } - } - - return result; + return function.apply(rve); } @@ -1488,4 +1598,22 @@ public class AmbariCustomCommandExecutionHelper { hosts.removeAll(removedHosts); return removedHosts; } + + /** + * Class that is used to update base urls. There are two implementations of this - when we no + * longer are sure the deprecated repo info can be removed, so too can this class. + */ + @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) + abstract static class BaseUrlUpdater<T> implements Function<RepositoryVersionEntity, T> { + private T m_default; + + private BaseUrlUpdater(T defaultValue) { + m_default = defaultValue; + } + + private T getDefault() { + return m_default; + } + + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java index 751ce08..04b72ea 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java @@ -2356,7 +2356,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle commandParams.put(ExecutionCommand.KeyNames.REFRESH_TOPOLOGY, "True"); } - String repoInfo = customCommandExecutionHelper.getRepoInfo(cluster, hostEntity.getOsType(), osFamily , hostname); + String repoInfo = customCommandExecutionHelper.getRepoInfo(cluster, host); if (LOG.isDebugEnabled()) { LOG.debug("Sending repo information to agent" + ", hostname=" + scHost.getHostName() @@ -2472,7 +2472,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle execCmd.setCommandParams(commandParams); execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion())); - + execCmd.setRepositoryFile(customCommandExecutionHelper.getCommandRepository(cluster, host)); if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) { LOG.debug("AmbariManagementControllerImpl.createHostAction: created ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, with cluster-env tags {}", http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java index e39588e..c3e66fc 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProvider.java @@ -29,6 +29,8 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.ambari.annotations.Experimental; +import org.apache.ambari.annotations.ExperimentalFeature; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.StaticallyInject; @@ -41,7 +43,6 @@ import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; import org.apache.ambari.server.agent.CommandReport; import org.apache.ambari.server.agent.ExecutionCommand; -import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.ActionExecutionContext; @@ -79,10 +80,9 @@ import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.RepositoryType; import org.apache.ambari.server.state.RepositoryVersionState; import org.apache.ambari.server.state.ServiceComponentHost; -import org.apache.ambari.server.state.ServiceInfo; -import org.apache.ambari.server.state.ServiceOsSpecific; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.repository.VersionDefinitionXml; +import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; import org.apache.ambari.server.utils.StageUtils; import org.apache.ambari.server.utils.VersionUtils; import org.apache.commons.lang.StringUtils; @@ -91,7 +91,6 @@ import org.apache.hadoop.metrics2.sink.relocated.google.common.collect.Lists; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; -import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Provider; @@ -126,7 +125,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou protected static final String CLUSTER_STACK_VERSION_FORCE = "ClusterStackVersions/force"; protected static final String INSTALL_PACKAGES_ACTION = "install_packages"; - protected static final String INSTALL_PACKAGES_FULL_NAME = "Install version"; + protected static final String INSTALL_PACKAGES_FULL_NAME = "Install Version"; /** * The default success factor that will be used when determining if a stage's @@ -173,9 +172,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou private static HostRoleCommandFactory hostRoleCommandFactory; @Inject - private static Gson gson; - - @Inject private static Provider<AmbariActionExecutionHelper> actionExecutionHelper; @Inject @@ -193,6 +189,9 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou @Inject private static HostComponentStateDAO hostComponentStateDAO; + @Inject + private static RepositoryVersionHelper repoVersionHelper; + /** * We have to include such a hack here, because if we * make finalizeUpgradeAction field static and request injection @@ -602,6 +601,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou throws SystemException { // Determine repositories for host String osFamily = host.getOsFamily(); + final List<RepositoryEntity> repoInfo = perOsRepos.get(osFamily); if (repoInfo == null) { throw new SystemException(String.format("Repositories for os type %s are " + @@ -614,7 +614,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou } // determine packages for all services that are installed on host - List<ServiceOsSpecific.Package> packages = new ArrayList<>(); Set<String> servicesOnHost = new HashSet<>(); List<ServiceComponentHost> components = cluster.getServiceComponentHosts(host.getHostName()); for (ServiceComponentHost component : components) { @@ -627,65 +626,9 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou return null; } - List<String> blacklistedPackagePrefixes = configuration.getRollingUpgradeSkipPackagesPrefixes(); - for (String serviceName : servicesOnHost) { - try{ - if(ami.isServiceRemovedInStack(stackId.getStackName(), stackId.getStackVersion(), serviceName)){ - LOG.info(String.format("%s has been removed from stack %s-%s. Skip calculating its installation packages", stackId.getStackName(), stackId.getStackVersion(), serviceName)); - continue; //No need to calculate install packages for removed services - } - } catch (AmbariException e1) { - throw new SystemException(String.format("Cannot obtain stack information for %s-%s", stackId.getStackName(), stackId.getStackVersion()), e1); - } - - ServiceInfo info; - try { - info = ami.getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); - } catch (AmbariException e) { - throw new SystemException("Cannot enumerate services", e); - } - - List<ServiceOsSpecific.Package> packagesForService = managementController.getPackagesForServiceHost(info, - new HashMap<String, String>(), // Contents are ignored - osFamily); - for (ServiceOsSpecific.Package aPackage : packagesForService) { - if (! aPackage.getSkipUpgrade()) { - boolean blacklisted = false; - for(String prefix : blacklistedPackagePrefixes) { - if (aPackage.getName().startsWith(prefix)) { - blacklisted = true; - break; - } - } - if (! blacklisted) { - packages.add(aPackage); - } - } - } - } - - final String packageList = gson.toJson(packages); - final String repoList = gson.toJson(repoInfo); - - Map<String, String> params = new HashMap<>(); - params.put("stack_id", stackId.getStackId()); - params.put("repository_version", repoVersion.getVersion()); - params.put("base_urls", repoList); - params.put(KeyNames.PACKAGE_LIST, packageList); - params.put(KeyNames.REPO_VERSION_ID, repoVersion.getId().toString()); - - VersionDefinitionXml xml = null; - try { - xml = repoVersion.getRepositoryXml(); - } catch (Exception e) { - throw new SystemException(String.format("Could not load xml from repo version %s", - repoVersion.getVersion())); - } - - if (null != xml && StringUtils.isNotBlank(xml.getPackageVersion(osFamily))) { - params.put(KeyNames.PACKAGE_VERSION, xml.getPackageVersion(osFamily)); - } + Map<String, String> roleParams = repoVersionHelper.buildRoleParams(managementController, repoVersion, + osFamily, servicesOnHost); // add host to this stage RequestResourceFilter filter = new RequestResourceFilter(null, null, @@ -694,9 +637,11 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou ActionExecutionContext actionContext = new ActionExecutionContext( cluster.getClusterName(), INSTALL_PACKAGES_ACTION, Collections.singletonList(filter), - params); + roleParams); actionContext.setTimeout(Short.valueOf(configuration.getDefaultAgentTaskTimeout(true))); + repoVersionHelper.addCommandRepository(actionContext, osFamily, repoVersion, repoInfo); + return actionContext; } @@ -734,29 +679,6 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou return false; } - - /** - * Sets host versions states to not-required. - * - * Transitioning host version to NOT_REQUIRED state manually is ok since - * other completion handlers set success/fail states correctly during heartbeat. - * The number of NOT_REQUIRED components for a cluster will be low. - */ - private void transitionHostVersionToNotRequired(Host host, Cluster cluster, RepositoryVersionEntity repoVersion) { - LOG.info(String.format("Transitioning version %s on host %s directly to %s" + - " without distributing bits to host since it has no versionable components.", - repoVersion.getVersion(), host.getHostName(), RepositoryVersionState.NOT_REQUIRED)); - - for (HostVersionEntity hve : host.getAllHostVersions()) { - if (hve.getRepositoryVersion().equals(repoVersion)) { - hve.setState(RepositoryVersionState.NOT_REQUIRED); - hostVersionDAO.merge(hve); - } - } - - } - - private RequestStageContainer createRequest() { ActionManager actionManager = getManagementController().getActionManager(); @@ -876,6 +798,7 @@ public class ClusterStackVersionResourceProvider extends AbstractControllerResou // !!! revisit for PU // If forcing to become CURRENT, get the Cluster Version whose state is CURRENT and make sure that // the Host Version records for the same Repo Version are also marked as CURRENT. + @Experimental(feature=ExperimentalFeature.PATCH_UPGRADES) ClusterVersionEntity current = cluster.getCurrentClusterVersion(); if (!current.getRepositoryVersion().equals(rve)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java index 811ce9b..71b9b6b 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProvider.java @@ -19,7 +19,6 @@ package org.apache.ambari.server.controller.internal; import static org.apache.ambari.server.agent.ExecutionCommand.KeyNames.JDK_LOCATION; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -34,7 +33,6 @@ import org.apache.ambari.server.actionmanager.ActionManager; import org.apache.ambari.server.actionmanager.RequestFactory; import org.apache.ambari.server.actionmanager.Stage; import org.apache.ambari.server.actionmanager.StageFactory; -import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.ActionExecutionContext; @@ -61,17 +59,14 @@ import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.Host; import org.apache.ambari.server.state.RepositoryVersionState; import org.apache.ambari.server.state.ServiceComponentHost; -import org.apache.ambari.server.state.ServiceInfo; -import org.apache.ambari.server.state.ServiceOsSpecific; import org.apache.ambari.server.state.StackId; -import org.apache.ambari.server.state.repository.VersionDefinitionXml; +import org.apache.ambari.server.state.stack.upgrade.RepositoryVersionHelper; import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.Validate; import com.google.common.collect.Sets; -import com.google.gson.Gson; import com.google.inject.Inject; import com.google.inject.Provider; @@ -107,7 +102,7 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource protected static final String INSTALL_PACKAGES_ACTION = "install_packages"; protected static final String STACK_SELECT_ACTION = "ru_set_all"; - protected static final String INSTALL_PACKAGES_FULL_NAME = "Install version"; + protected static final String INSTALL_PACKAGES_FULL_NAME = "Install Version"; private static Set<String> pkPropertyIds = Sets.newHashSet( @@ -147,8 +142,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource @Inject private static RepositoryVersionDAO repositoryVersionDAO; - private static Gson gson = StageUtils.getGson(); - @Inject private static StageFactory stageFactory; @@ -161,6 +154,8 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource @Inject private static Configuration configuration; + @Inject + private static RepositoryVersionHelper repoVersionHelper; /** * Constructor. @@ -403,8 +398,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource LOG.error(String.format("Repository list is empty. Ambari may not be managing the repositories for %s", osFamily)); } - // For every host at cluster, determine packages for all installed services - List<ServiceOsSpecific.Package> packages = new ArrayList<>(); Set<String> servicesOnHost = new HashSet<>(); if (forceInstallOnNonMemberHost) { @@ -426,7 +419,6 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource LOG.error("Service not found for component {}!", componentName, e); throw new IllegalArgumentException("Service not found for component : " + componentName); } - } } else { @@ -435,53 +427,9 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource servicesOnHost.add(component.getServiceName()); } } - List<String> blacklistedPackagePrefixes = configuration.getRollingUpgradeSkipPackagesPrefixes(); - for (String serviceName : servicesOnHost) { - ServiceInfo info; - try { - info = ami.getService(stackName, stackVersion, serviceName); - } catch (AmbariException e) { - throw new SystemException("Can not enumerate services", e); - } - List<ServiceOsSpecific.Package> packagesForService = managementController.getPackagesForServiceHost(info, - new HashMap<String, String>(), // Contents are ignored - osFamily); - for (ServiceOsSpecific.Package aPackage : packagesForService) { - if (! aPackage.getSkipUpgrade()) { - boolean blacklisted = false; - for(String prefix : blacklistedPackagePrefixes) { - if (aPackage.getName().startsWith(prefix)) { - blacklisted = true; - break; - } - } - if (! blacklisted) { - packages.add(aPackage); - } - } - } - } - final String packageList = gson.toJson(packages); - final String repoList = gson.toJson(repoInfo); - - Map<String, String> params = new HashMap<String, String>(){{ - put("stack_id", stackId.getStackId()); - put("repository_version", desiredRepoVersion); - put("base_urls", repoList); - put(KeyNames.PACKAGE_LIST, packageList); - }}; - - VersionDefinitionXml xml = null; - try { - xml = repoVersionEnt.getRepositoryXml(); - } catch (Exception e) { - throw new SystemException(String.format("Could not load xml from repo version %s", - repoVersionEnt.getVersion())); - } - if (null != xml && StringUtils.isNotBlank(xml.getPackageVersion(osFamily))) { - params.put(KeyNames.PACKAGE_VERSION, xml.getPackageVersion(osFamily)); - } + Map<String, String> roleParams = repoVersionHelper.buildRoleParams(managementController, repoVersionEnt, + osFamily, servicesOnHost); // Create custom action RequestResourceFilter filter = new RequestResourceFilter(null, null, @@ -490,9 +438,11 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource ActionExecutionContext actionContext = new ActionExecutionContext( cluster.getClusterName(), INSTALL_PACKAGES_ACTION, Collections.singletonList(filter), - params); + roleParams); actionContext.setTimeout(Short.valueOf(configuration.getDefaultAgentTaskTimeout(true))); + repoVersionHelper.addCommandRepository(actionContext, osFamily, repoVersionEnt, repoInfo); + String caption = String.format(INSTALL_PACKAGES_FULL_NAME + " on host %s", hostName); RequestStageContainer req = createRequest(caption); @@ -553,7 +503,7 @@ public class HostStackVersionResourceProvider extends AbstractControllerResource Stage stage; long stageId; ActionExecutionContext actionContext; - Map<String, String> commandParams = new HashMap(); + Map<String, String> commandParams = new HashMap<>(); commandParams.put("version", desiredRepoVersion); stage = stageFactory.createNew(req.getId(), http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java index fdb7c8d..14e0179 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelper.java @@ -19,18 +19,33 @@ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.agent.CommandRepository; +import org.apache.ambari.server.agent.ExecutionCommand; +import org.apache.ambari.server.agent.ExecutionCommand.KeyNames; import org.apache.ambari.server.api.services.AmbariMetaInfo; +import org.apache.ambari.server.configuration.Configuration; +import org.apache.ambari.server.controller.ActionExecutionContext; +import org.apache.ambari.server.controller.ActionExecutionContext.ExecutionCommandVisitor; +import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.internal.OperatingSystemResourceProvider; import org.apache.ambari.server.controller.internal.RepositoryResourceProvider; import org.apache.ambari.server.controller.internal.RepositoryVersionResourceProvider; +import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.orm.entities.OperatingSystemEntity; import org.apache.ambari.server.orm.entities.RepositoryEntity; +import org.apache.ambari.server.orm.entities.RepositoryVersionEntity; import org.apache.ambari.server.state.RepositoryInfo; +import org.apache.ambari.server.state.ServiceInfo; +import org.apache.ambari.server.state.ServiceOsSpecific; +import org.apache.ambari.server.state.StackId; +import org.apache.ambari.server.state.repository.VersionDefinitionXml; import org.apache.ambari.server.state.stack.UpgradePack; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -44,6 +59,7 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.google.inject.Inject; +import com.google.inject.Provider; import com.google.inject.Singleton; /** @@ -57,8 +73,11 @@ public class RepositoryVersionHelper { @Inject private Gson gson; - @Inject(optional = true) - private AmbariMetaInfo ambariMetaInfo; + @Inject + private Provider<AmbariMetaInfo> ami; + + @Inject + private Provider<Configuration> configuration; /** * Parses operating systems json to a list of entities. Expects json like: @@ -198,7 +217,7 @@ public class RepositoryVersionHelper { * @throws AmbariException if no upgrade packs suit the requirements */ public String getUpgradePackageName(String stackName, String stackVersion, String repositoryVersion, UpgradeType upgradeType) throws AmbariException { - final Map<String, UpgradePack> upgradePacks = ambariMetaInfo.getUpgradePacks(stackName, stackVersion); + final Map<String, UpgradePack> upgradePacks = ami.get().getUpgradePacks(stackName, stackVersion); for (UpgradePack upgradePack : upgradePacks.values()) { final String upgradePackName = upgradePack.getName(); @@ -218,4 +237,109 @@ public class RepositoryVersionHelper { throw new AmbariException("There were no suitable upgrade packs for stack " + stackName + " " + stackVersion + ((null != upgradeType) ? " and upgrade type " + upgradeType : "")); } + + /** + * Build the role parameters for an install command. + * + * @param amc the management controller. Tests don't use the same instance that gets injected. + * @param repoVersion the repository version + * @param osFamily the os family + * @param services the set of services to check for packages + * @param repos the list of individual repositories + * @return a Map<String, String> to use in + * @throws SystemException + */ + public Map<String, String> buildRoleParams(AmbariManagementController amc, RepositoryVersionEntity repoVersion, String osFamily, Set<String> servicesOnHost) + throws SystemException { + + StackId stackId = repoVersion.getStackId(); + + List<ServiceOsSpecific.Package> packages = new ArrayList<>(); + + for (String serviceName : servicesOnHost) { + ServiceInfo info; + + try { + if (ami.get().isServiceRemovedInStack(stackId.getStackName(), stackId.getStackVersion(), serviceName)) { + LOG.info(String.format("%s has been removed from stack %s-%s. Skip calculating its installation packages", stackId.getStackName(), stackId.getStackVersion(), serviceName)); + continue; //No need to calculate install packages for removed services + } + + info = ami.get().getService(stackId.getStackName(), stackId.getStackVersion(), serviceName); + } catch (AmbariException e) { + throw new SystemException(String.format("Cannot obtain stack information for %s-%s", stackId.getStackName(), stackId.getStackVersion()), e); + } + + List<ServiceOsSpecific.Package> packagesForService = amc.getPackagesForServiceHost(info, + new HashMap<String, String>(), osFamily); + + List<String> blacklistedPackagePrefixes = configuration.get().getRollingUpgradeSkipPackagesPrefixes(); + + for (ServiceOsSpecific.Package aPackage : packagesForService) { + if (!aPackage.getSkipUpgrade()) { + boolean blacklisted = false; + for (String prefix : blacklistedPackagePrefixes) { + if (aPackage.getName().startsWith(prefix)) { + blacklisted = true; + break; + } + } + if (! blacklisted) { + packages.add(aPackage); + } + } + } + } + + Map<String, String> roleParams = new HashMap<>(); + roleParams.put("stack_id", stackId.getStackId()); + roleParams.put("repository_version", repoVersion.getVersion()); + // !!! TODO make roleParams <String, Object> so we don't have to do this awfulness. + roleParams.put(KeyNames.PACKAGE_LIST, gson.toJson(packages)); + roleParams.put(KeyNames.REPO_VERSION_ID, repoVersion.getId().toString()); + + VersionDefinitionXml xml = null; + try { + xml = repoVersion.getRepositoryXml(); + } catch (Exception e) { + throw new SystemException(String.format("Could not load xml from repo version %s", + repoVersion.getVersion())); + } + + if (null != xml && StringUtils.isNotBlank(xml.getPackageVersion(osFamily))) { + roleParams.put(KeyNames.PACKAGE_VERSION, xml.getPackageVersion(osFamily)); + } + + return roleParams; + } + + /** + * Adds a command repository to the action context + * @param context the context + * @param osFamily the OS family + * @param repoVersion the repository version entity + * @param repos the repository entities + */ + public void addCommandRepository(ActionExecutionContext context, String osFamily, + RepositoryVersionEntity repoVersion, List<RepositoryEntity> repos) { + StackId stackId = repoVersion.getStackId(); + + final CommandRepository commandRepo = new CommandRepository(); + commandRepo.setRepositories(osFamily, repos); + commandRepo.setRepositoryVersion(repoVersion.getVersion()); + commandRepo.setRepositoryVersionId(repoVersion.getId()); + commandRepo.setStackName(stackId.getStackName()); + commandRepo.setUniqueSuffix(String.format("-repo-%s", repoVersion.getId())); + + context.addVisitor(new ExecutionCommandVisitor() { + @Override + public void visit(ExecutionCommand command) { + if (null == command.getRepositoryFile()) { + command.setRepositoryFile(commandRepo); + } + } + }); + } + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py index 33adce1..c12dff1 100644 --- a/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py +++ b/ambari-server/src/main/resources/custom_actions/scripts/install_packages.py @@ -16,36 +16,30 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. -Ambari Agent - """ -import os import signal import re -import os.path -import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. +import ambari_simplejson as json -from resource_management import * -import resource_management -from resource_management.libraries.functions.list_ambari_managed_repos import list_ambari_managed_repos -from ambari_commons.os_check import OSCheck, OSConst +from ambari_commons.os_check import OSCheck from ambari_commons.str_utils import cbool, cint +from resource_management.core.exceptions import Fail +from resource_management.core.logger import Logger +from resource_management.core.resources import Package from resource_management.libraries.functions.packages_analyzer import allInstalledPackages, verifyDependencies from resource_management.libraries.functions import conf_select from resource_management.libraries.functions import stack_tools from resource_management.libraries.functions.stack_select import get_stack_versions from resource_management.libraries.functions.version import format_stack_version from resource_management.libraries.functions.repo_version_history \ - import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE -from resource_management.libraries.script.script import Script -from resource_management.core.resources.system import Execute -from resource_management.libraries.functions.stack_features import check_stack_feature + import read_actual_version_from_history_file, write_actual_version_to_history_file, REPO_VERSION_HISTORY_FILE from resource_management.libraries.functions import StackFeature - -from resource_management.core.logger import Logger - +from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository +from resource_management.libraries.functions.stack_features import check_stack_feature +from resource_management.libraries.resources.repository import Repository +from resource_management.libraries.script.script import Script class InstallPackages(Script): """ @@ -73,22 +67,20 @@ class InstallPackages(Script): self.repository_version_id = None + base_urls = [] # Select dict that contains parameters try: + if 'base_urls' in config['roleParams']: + base_urls = json.loads(config['roleParams']['base_urls']) + self.repository_version = config['roleParams']['repository_version'] - base_urls = json.loads(config['roleParams']['base_urls']) package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] + if 'repository_version_id' in config['roleParams']: self.repository_version_id = config['roleParams']['repository_version_id'] except KeyError: - # Last try - self.repository_version = config['commandParams']['repository_version'] - base_urls = json.loads(config['commandParams']['base_urls']) - package_list = json.loads(config['commandParams']['package_list']) - stack_id = config['commandParams']['stack_id'] - if 'repository_version_id' in config['commandParams']: - self.repository_version_id = config['commandParams']['repository_version_id'] + pass # current stack information self.current_stack_version_formatted = None @@ -110,9 +102,7 @@ class InstallPackages(Script): self.repository_version = self.repository_version.strip() - # Install/update repositories - installed_repositories = [] self.current_repositories = [] self.current_repo_files = set() @@ -130,21 +120,22 @@ class InstallPackages(Script): Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version)) try: - append_to_file = False - for url_info in base_urls: - repo_name, repo_file = self.install_repository(url_info, append_to_file, template) - self.current_repositories.append(repo_name) - self.current_repo_files.add(repo_file) - append_to_file = True - - installed_repositories = list_ambari_managed_repos(self.stack_name) + if 'repositoryFile' in config: + create_repo_files(template, CommandRepository(config['repositoryFile'])) + else: + append_to_file = False + for url_info in base_urls: + repo_name, repo_file = self.install_repository(url_info, append_to_file, template) + self.current_repositories.append(repo_name) + self.current_repo_files.add(repo_file) + append_to_file = True + except Exception, err: - Logger.logger.exception("Cannot distribute repositories. Error: {0}".format(str(err))) + Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1 # Build structured output with initial values self.structured_output = { - 'ambari_repositories': installed_repositories, 'installed_repository_version': self.repository_version, 'stack_id': stack_id, 'package_installation_result': 'FAIL' http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py index 6193c11..50c5a40 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py @@ -111,3 +111,5 @@ if has_hbase_masters: #repo params repo_info = config['hostLevelParams']['repo_info'] service_repo_info = default("/hostLevelParams/service_repo_info",None) + +repo_file = default("/repositoryFile", None) http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py index a35dce7..cbbc466 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py +++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/repo_initialization.py @@ -19,11 +19,10 @@ limitations under the License. from ambari_commons.os_check import OSCheck from resource_management.libraries.resources.repository import Repository +from resource_management.libraries.functions.repository_util import create_repo_files, CommandRepository, UBUNTU_REPO_COMPONENTS_POSTFIX from resource_management.core.logger import Logger -import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set. +import ambari_simplejson as json -# components_lits = repoName + postfix -_UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"] def _alter_repo(action, repo_string, repo_template): """ @@ -45,17 +44,17 @@ def _alter_repo(action, repo_string, repo_template): repo['baseUrl'] = None if not 'mirrorsList' in repo: repo['mirrorsList'] = None - - ubuntu_components = [ repo['repoName'] ] + _UBUNTU_REPO_COMPONENTS_POSTFIX - + + ubuntu_components = [ repo['repoName'] ] + UBUNTU_REPO_COMPONENTS_POSTFIX + Repository(repo['repoId'], action = action, base_url = repo['baseUrl'], mirror_list = repo['mirrorsList'], repo_file_name = repo['repoName'], repo_template = repo_template, - components = ubuntu_components, # ubuntu specific - ) + components = ubuntu_components) # ubuntu specific + def install_repos(): import params @@ -63,6 +62,13 @@ def install_repos(): return template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu + + # use this newer way of specifying repositories, if available + if params.repo_file is not None: + create_repo_files(template, CommandRepository(params.repo_file)) + return + _alter_repo("create", params.repo_info, template) + if params.service_repo_info: _alter_repo("create", params.service_repo_info, template) http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java index 8f7b31d..6245708 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/ClusterStackVersionResourceProviderTest.java @@ -219,9 +219,14 @@ public class ClusterStackVersionResourceProviderTest { Map<String, String> hostLevelParams = new HashMap<>(); StackId stackId = new StackId("HDP", "2.0.1"); + StackEntity stackEntity = new StackEntity(); + stackEntity.setStackName("HDP"); + stackEntity.setStackVersion("2.1.1"); + RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); repoVersion.setId(1l); repoVersion.setOperatingSystems(OS_JSON); + repoVersion.setStack(stackEntity); final String hostWithoutVersionableComponents = "host2"; @@ -647,12 +652,17 @@ public class ClusterStackVersionResourceProviderTest { File f = new File("src/test/resources/hbase_version_test.xml"); + StackEntity stackEntity = new StackEntity(); + stackEntity.setStackName("HDP"); + stackEntity.setStackVersion("2.1.1"); + RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); repoVersion.setId(1l); repoVersion.setOperatingSystems(OS_JSON); repoVersion.setVersionXml(IOUtils.toString(new FileInputStream(f))); repoVersion.setVersionXsd("version_definition.xsd"); repoVersion.setType(RepositoryType.STANDARD); + repoVersion.setStack(stackEntity); ambariMetaInfo.getComponent("HDP", "2.1.1", "HBASE", "HBASE_MASTER").setVersionAdvertised(true); @@ -890,12 +900,17 @@ public class ClusterStackVersionResourceProviderTest { File f = new File("src/test/resources/hbase_version_test.xml"); + StackEntity stackEntity = new StackEntity(); + stackEntity.setStackName("HDP"); + stackEntity.setStackVersion("2.1.1"); + RepositoryVersionEntity repoVersion = new RepositoryVersionEntity(); repoVersion.setId(1l); repoVersion.setOperatingSystems(os_json); repoVersion.setVersionXml(IOUtils.toString(new FileInputStream(f))); repoVersion.setVersionXsd("version_definition.xsd"); repoVersion.setType(RepositoryType.STANDARD); + repoVersion.setStack(stackEntity); ambariMetaInfo.getComponent("HDP", "2.1.1", "HBASE", "HBASE_MASTER").setVersionAdvertised(true); @@ -1096,8 +1111,8 @@ public class ClusterStackVersionResourceProviderTest { Assert.assertEquals(Float.valueOf(0.85f), successFactor); Assert.assertTrue(executionCommand.getRoleParams().containsKey(KeyNames.PACKAGE_VERSION)); - Assert.assertTrue(executionCommand.getRoleParams().containsKey("base_urls")); - Assert.assertEquals("[]", executionCommand.getRoleParams().get("base_urls")); + Assert.assertNotNull(executionCommand.getRepositoryFile()); + Assert.assertEquals(0, executionCommand.getRepositoryFile().getRepositories().size()); } @Test http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java index 4bed482..1ae83d4 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HostStackVersionResourceProviderTest.java @@ -154,9 +154,10 @@ public class HostStackVersionResourceProviderTest { StackEntity stack = new StackEntity(); stack.setStackName("HDP"); - stack.setStackVersion("2.2"); + stack.setStackVersion("2.0.1"); repoVersion.setStack(stack); repoVersion.setVersion("2.2"); + repoVersion.setId(1L); } @After http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java index c6ee875..5513271 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/UpgradeActionTest.java @@ -40,6 +40,7 @@ import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; import org.apache.ambari.server.actionmanager.HostRoleStatus; import org.apache.ambari.server.agent.CommandReport; +import org.apache.ambari.server.agent.CommandRepository; import org.apache.ambari.server.agent.ExecutionCommand; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.controller.AmbariCustomCommandExecutionHelper; @@ -632,9 +633,6 @@ public class UpgradeActionTest { CommandReport report = finalizeUpgradeAction.execute(null); assertNotNull(report); assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); - - // Verify the metainfo url - verifyBaseRepoURL(helper, cluster, host, "http://foo1";); } /** @@ -699,6 +697,7 @@ public class UpgradeActionTest { } private void verifyBaseRepoURL(AmbariCustomCommandExecutionHelper helper, Cluster cluster, Host host, String expectedRepoBaseURL) throws AmbariException { + String repoInfo = helper.getRepoInfo(cluster, host); Gson gson = new Gson(); JsonElement element = gson.fromJson(repoInfo, JsonElement.class); @@ -709,6 +708,13 @@ public class UpgradeActionTest { JsonObject o = list.get(0).getAsJsonObject(); assertTrue(o.has("baseUrl")); assertEquals(expectedRepoBaseURL, o.get("baseUrl").getAsString()); + + CommandRepository commandRepo = helper.getCommandRepository(cluster, host); + + assertNotNull(commandRepo); + assertNotNull(commandRepo.getRepositories()); + assertEquals(1, commandRepo.getRepositories().size()); + assertEquals(expectedRepoBaseURL, commandRepo.getRepositories().iterator().next().getBaseUrl()); } @Test @@ -1019,9 +1025,6 @@ public class UpgradeActionTest { assertNotNull(report); assertEquals(HostRoleStatus.COMPLETED.name(), report.getStatus()); - // Verify the metainfo url - verifyBaseRepoURL(helper, cluster, host, "http://foo1";); - // ensure that history now exists historyEntites = serviceComponentDesiredStateDAO.findHistory(cluster.getClusterId(), nnSCH.getServiceName(), nnSCH.getServiceComponentName()); http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java b/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java index df69f68..d67cdfc 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/stack/UpdateActiveRepoVersionOnStartupTest.java @@ -24,6 +24,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import java.io.IOException; +import java.lang.reflect.Field; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.orm.InMemoryDefaultTestModule; @@ -45,6 +46,7 @@ import org.junit.Test; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; import com.google.common.io.Resources; +import com.google.gson.Gson; import com.google.inject.Guice; import com.google.inject.Injector; import com.google.inject.Provider; @@ -97,8 +99,14 @@ public class UpdateActiveRepoVersionOnStartupTest { ClusterDAO clusterDao = mock(ClusterDAO.class); ClusterVersionDAO clusterVersionDAO = mock(ClusterVersionDAO.class); repositoryVersionDao = mock(RepositoryVersionDAO.class); + final RepositoryVersionHelper repositoryVersionHelper = new RepositoryVersionHelper(); - AmbariMetaInfo metaInfo = mock(AmbariMetaInfo.class); + Field field = RepositoryVersionHelper.class.getDeclaredField("gson"); + field.setAccessible(true); + field.set(repositoryVersionHelper, new Gson()); + + + final AmbariMetaInfo metaInfo = mock(AmbariMetaInfo.class); StackManager stackManager = mock(StackManager.class); when(metaInfo.getStackManager()).thenReturn(stackManager); @@ -123,15 +131,26 @@ public class UpdateActiveRepoVersionOnStartupTest { stackInfo.getRepositories().add(repositoryInfo); when(stackManager.getStack("HDP", "2.3")).thenReturn(stackInfo); - Provider<RepositoryVersionHelper> repositoryVersionHelperProvider = mock(Provider.class); + final Provider<RepositoryVersionHelper> repositoryVersionHelperProvider = mock(Provider.class); when(repositoryVersionHelperProvider.get()).thenReturn(repositoryVersionHelper); + + + InMemoryDefaultTestModule testModule = new InMemoryDefaultTestModule() { @Override protected void configure() { - bind(RepositoryVersionHelper.class).toInstance(repositoryVersionHelper); + bind(RepositoryVersionHelper.class).toProvider(repositoryVersionHelperProvider); + bind(AmbariMetaInfo.class).toProvider(new Provider<AmbariMetaInfo>() { + @Override + public AmbariMetaInfo get() { + return metaInfo; + } + }); + requestStaticInjection(RepositoryVersionEntity.class); } }; + Injector injector = Guice.createInjector(testModule); if (addClusterVersion) { repoVersion = new RepositoryVersionEntity(); http://git-wip-us.apache.org/repos/asf/ambari/blob/b6866249/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java index 3a5532b..339786e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/state/stack/upgrade/RepositoryVersionHelperTest.java @@ -17,19 +17,15 @@ */ package org.apache.ambari.server.state.stack.upgrade; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import org.apache.ambari.server.state.RepositoryInfo; -import org.junit.Before; +import org.junit.Assert; import org.junit.Test; import com.google.gson.Gson; -import com.google.inject.AbstractModule; -import com.google.inject.Guice; -import com.google.inject.Injector; - -import junit.framework.Assert; /** * Tests the {@link RepositoryVersionHelper} class @@ -38,20 +34,15 @@ public class RepositoryVersionHelperTest { private RepositoryVersionHelper helper; - @Before - public void before() throws Exception { - final Injector injector = Guice.createInjector(new AbstractModule() { - - @Override - protected void configure() { - bind(Gson.class).toInstance(new Gson()); - } - }); - helper = injector.getInstance(RepositoryVersionHelper.class); - } - @Test public void testSerializeOperatingSystems() throws Exception { + Gson gson = new Gson(); + Field field = RepositoryVersionHelper.class.getDeclaredField("gson"); + field.setAccessible(true); + + RepositoryVersionHelper helper = new RepositoryVersionHelper(); + field.set(helper, gson); + final List<RepositoryInfo> repositories = new ArrayList<>(); final RepositoryInfo repository = new RepositoryInfo(); repository.setBaseUrl("baseurl");
