This is an automated email from the ASF dual-hosted git repository. swagle pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new 6f79c88 [AMBARI-23370] Download client configs fails due to 'clusterLevelParams' not found (#802) 6f79c88 is described below commit 6f79c888aed062ef60d40bbd647d43f5629b0b2d Author: majorendre <34535487+majoren...@users.noreply.github.com> AuthorDate: Thu Mar 29 20:55:37 2018 +0200 [AMBARI-23370] Download client configs fails due to 'clusterLevelParams' not found (#802) --- .../server/controller/KerberosHelperImpl.java | 10 +--- .../internal/ClientConfigResourceProvider.java | 61 +++++++--------------- .../topology/ClusterConfigurationRequest.java | 5 +- .../org/apache/ambari/server/utils/StageUtils.java | 61 +++++----------------- .../controller/AmbariManagementControllerTest.java | 2 +- .../apache/ambari/server/utils/StageUtilsTest.java | 24 ++++----- 6 files changed, 45 insertions(+), 118 deletions(-) diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java index d2323c4..3f0ea84 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/KerberosHelperImpl.java @@ -588,8 +588,6 @@ public class KerberosHelperImpl implements KerberosHelper { configurations.put("clusterHostInfo", clusterHostInfoMap); } - Map<String, String> componentToClusterInfoMap = StageUtils.getComponentToClusterInfoKeyMap(); - // Iterate through the recommendations to find the recommended host assignments for (RecommendationResponse.HostGroup hostGroup : hostGroups) { Set<Map<String, String>> components = hostGroup.getComponents(); @@ -607,13 +605,7 @@ public class KerberosHelperImpl implements KerberosHelper { // If the component filter is null or the current component is found in the filter, // include it in the map if ((componentFilter == null) || componentFilter.contains(componentName)) { - String key = componentToClusterInfoMap.get(componentName); - - if (StringUtils.isEmpty(key)) { - // If not found in the componentToClusterInfoMap, then keys are assumed to be - // in the form of <component_name>_hosts (lowercase) - key = componentName.toLowerCase() + "_hosts"; - } + String key = StageUtils.getClusterHostInfoKey(componentName); Set<String> fqdns = new TreeSet<>(); diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java index fda3817..8992413 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java @@ -65,6 +65,7 @@ import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; +import org.apache.ambari.server.controller.AmbariManagementControllerImpl; import org.apache.ambari.server.controller.MaintenanceStateHelper; import org.apache.ambari.server.controller.ServiceComponentHostRequest; import org.apache.ambari.server.controller.ServiceComponentHostResponse; @@ -369,27 +370,13 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv } osFamily = clusters.getHost(hostName).getOsFamily(); - TreeMap<String, String> hostLevelParams = new TreeMap<>(); - StageUtils.useStackJdkIfExists(hostLevelParams, configs); - hostLevelParams.put(JDK_LOCATION, managementController.getJdkResourceUrl()); - hostLevelParams.put(STACK_NAME, stackId.getStackName()); - hostLevelParams.put(STACK_VERSION, stackId.getStackVersion()); - hostLevelParams.put(DB_NAME, managementController.getServerDB()); - hostLevelParams.put(MYSQL_JDBC_URL, managementController.getMysqljdbcUrl()); - hostLevelParams.put(ORACLE_JDBC_URL, managementController.getOjdbcUrl()); - hostLevelParams.put(HOST_SYS_PREPPED, configs.areHostsSysPrepped()); - hostLevelParams.putAll(managementController.getRcaParameters()); - hostLevelParams.put(AGENT_STACK_RETRY_ON_UNAVAILABILITY, configs.isAgentStackRetryOnInstallEnabled()); - hostLevelParams.put(AGENT_STACK_RETRY_COUNT, configs.getAgentStackRetryOnInstallCount()); - hostLevelParams.put(GPL_LICENSE_ACCEPTED, configs.getGplLicenseAccepted().toString()); - // Write down os specific info for the service ServiceOsSpecific anyOs = null; if (serviceInfo.getOsSpecifics().containsKey(AmbariMetaInfo.ANY_OS)) { anyOs = serviceInfo.getOsSpecifics().get(AmbariMetaInfo.ANY_OS); } - ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo, hostLevelParams, osFamily); + ServiceOsSpecific hostOs = populateServicePackagesInfo(serviceInfo, osFamily); // Build package list that is relevant for host List<ServiceOsSpecific.Package> packages = @@ -403,25 +390,6 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv } String packageList = gson.toJson(packages); - Set<String> userSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.USER, cluster, desiredClusterConfigs); - String userList = gson.toJson(userSet); - hostLevelParams.put(USER_LIST, userList); - - //Create a user_group mapping and send it as part of the hostLevelParams - Map<String, Set<String>> userGroupsMap = configHelper.createUserGroupsMap( - stackId, cluster, desiredClusterConfigs); - String userGroups = gson.toJson(userGroupsMap); - hostLevelParams.put(USER_GROUPS,userGroups); - - Set<String> groupSet = configHelper.getPropertyValuesWithPropertyType(stackId, PropertyType.GROUP, cluster, desiredClusterConfigs); - String groupList = gson.toJson(groupSet); - hostLevelParams.put(GROUP_LIST, groupList); - - Map<org.apache.ambari.server.state.PropertyInfo, String> notManagedHdfsPathMap = configHelper.getPropertiesWithPropertyType(stackId, PropertyType.NOT_MANAGED_HDFS_PATH, cluster, desiredClusterConfigs); - Set<String> notManagedHdfsPathSet = configHelper.filterInvalidPropertyValues(notManagedHdfsPathMap, NOT_MANAGED_HDFS_PATH_LIST); - String notManagedHdfsPathList = gson.toJson(notManagedHdfsPathSet); - hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList); - String jsonConfigurations = null; Map<String, Object> commandParams = new HashMap<>(); List<Map<String, String>> xmlConfigs = new LinkedList<>(); @@ -441,6 +409,18 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv } } + TreeMap<String, String> clusterLevelParams = null; + TreeMap<String, String> ambariLevelParams = null; + if (getManagementController() instanceof AmbariManagementControllerImpl){ + AmbariManagementControllerImpl controller = ((AmbariManagementControllerImpl)getManagementController()); + clusterLevelParams = controller.getMetadataClusterLevelParams(cluster, stackId); + ambariLevelParams = controller.getMetadataAmbariLevelParams(); + } + TreeMap<String, String> agentLevelParams = new TreeMap<>(); + agentLevelParams.put("hostname", hostName); + agentLevelParams.put("public_hostname", publicHostName); + agentLevelParams.put("agentCacheDir", "/var/lib/ambari-agent/cache"); + commandParams.put(PACKAGE_LIST, packageList); commandParams.put("xml_configs_list", xmlConfigs); commandParams.put("env_configs_list", envConfigs); @@ -452,7 +432,9 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv jsonContent.put("configuration_attributes", configurationAttributes); jsonContent.put("commandParams", commandParams); jsonContent.put("clusterHostInfo", clusterHostInfo); - jsonContent.put("hostLevelParams", hostLevelParams); + jsonContent.put("ambariLevelParams", ambariLevelParams); + jsonContent.put("clusterLevelParams", clusterLevelParams); + jsonContent.put("agentLevelParams", agentLevelParams); jsonContent.put("hostname", hostName); jsonContent.put("public_hostname", publicHostName); jsonContent.put("clusterName", cluster.getClusterName()); @@ -926,20 +908,13 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv } - protected ServiceOsSpecific populateServicePackagesInfo(ServiceInfo serviceInfo, Map<String, String> hostParams, - String osFamily) { + protected ServiceOsSpecific populateServicePackagesInfo(ServiceInfo serviceInfo, String osFamily) { ServiceOsSpecific hostOs = new ServiceOsSpecific(osFamily); List<ServiceOsSpecific> foundedOSSpecifics = getOSSpecificsByFamily(serviceInfo.getOsSpecifics(), osFamily); if (!foundedOSSpecifics.isEmpty()) { for (ServiceOsSpecific osSpecific : foundedOSSpecifics) { hostOs.addPackages(osSpecific.getPackages()); } - // Choose repo that is relevant for host - ServiceOsSpecific.Repo serviceRepo = hostOs.getRepo(); - if (serviceRepo != null) { - String serviceRepoInfo = gson.toJson(serviceRepo); - hostParams.put(SERVICE_REPO_INFO, serviceRepoInfo); - } } return hostOs; diff --git a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java index 740dd91..92aecb3 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/topology/ClusterConfigurationRequest.java @@ -285,10 +285,7 @@ public class ClusterConfigurationRequest { for (String component : components) { Collection<String> componentHost = clusterTopology.getHostAssignmentsForComponent(component); // retrieve corresponding clusterInfoKey for component using StageUtils - String clusterInfoKey = StageUtils.getComponentToClusterInfoKeyMap().get(component); - if (clusterInfoKey == null) { - clusterInfoKey = component.toLowerCase() + "_hosts"; - } + String clusterInfoKey = StageUtils.getClusterHostInfoKey(component); componentHostsMap.put(clusterInfoKey, StringUtils.join(componentHost, ",")); } } diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java index b6287e6..1130e3d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java @@ -92,8 +92,6 @@ public class StageUtils { protected static final String RACKS = "all_racks"; protected static final String IPV4_ADDRESSES = "all_ipv4_ips"; - private static Map<String, String> componentToClusterInfoKeyMap = - new HashMap<>(); private static Map<String, String> decommissionedToClusterInfoKeyMap = new HashMap<>(); private volatile static Gson gson; @@ -152,36 +150,6 @@ public class StageUtils { } static { - componentToClusterInfoKeyMap.put("NAMENODE", "namenode_host"); - componentToClusterInfoKeyMap.put("JOBTRACKER", "jtnode_host"); - componentToClusterInfoKeyMap.put("SECONDARY_NAMENODE", "snamenode_host"); - componentToClusterInfoKeyMap.put("RESOURCEMANAGER", "rm_host"); - componentToClusterInfoKeyMap.put("NODEMANAGER", "nm_hosts"); - componentToClusterInfoKeyMap.put("HISTORYSERVER", "hs_host"); - componentToClusterInfoKeyMap.put("JOURNALNODE", "journalnode_hosts"); - componentToClusterInfoKeyMap.put("ZKFC", "zkfc_hosts"); - componentToClusterInfoKeyMap.put("ZOOKEEPER_SERVER", "zookeeper_hosts"); - componentToClusterInfoKeyMap.put("FLUME_HANDLER", "flume_hosts"); - componentToClusterInfoKeyMap.put("HBASE_MASTER", "hbase_master_hosts"); - componentToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "hbase_rs_hosts"); - componentToClusterInfoKeyMap.put("HIVE_SERVER", "hive_server_host"); - componentToClusterInfoKeyMap.put("HIVE_METASTORE", "hive_metastore_host"); - componentToClusterInfoKeyMap.put("OOZIE_SERVER", "oozie_server"); - componentToClusterInfoKeyMap.put("WEBHCAT_SERVER", "webhcat_server_host"); - componentToClusterInfoKeyMap.put("MYSQL_SERVER", "hive_mysql_host"); - componentToClusterInfoKeyMap.put("DASHBOARD", "dashboard_host"); - componentToClusterInfoKeyMap.put("GANGLIA_SERVER", "ganglia_server_host"); - componentToClusterInfoKeyMap.put("DATANODE", "slave_hosts"); - componentToClusterInfoKeyMap.put("TASKTRACKER", "mapred_tt_hosts"); - componentToClusterInfoKeyMap.put("HBASE_REGIONSERVER", "hbase_rs_hosts"); - componentToClusterInfoKeyMap.put("ACCUMULO_MASTER", "accumulo_master_hosts"); - componentToClusterInfoKeyMap.put("ACCUMULO_MONITOR", "accumulo_monitor_hosts"); - componentToClusterInfoKeyMap.put("ACCUMULO_GC", "accumulo_gc_hosts"); - componentToClusterInfoKeyMap.put("ACCUMULO_TRACER", "accumulo_tracer_hosts"); - componentToClusterInfoKeyMap.put("ACCUMULO_TSERVER", "accumulo_tserver_hosts"); - } - - static { decommissionedToClusterInfoKeyMap.put("DATANODE", "decom_dn_hosts"); decommissionedToClusterInfoKeyMap.put("TASKTRACKER", "decom_tt_hosts"); decommissionedToClusterInfoKeyMap.put("NODEMANAGER", "decom_nm_hosts"); @@ -192,10 +160,6 @@ public class StageUtils { return requestId + "-" + stageId; } - public static Map<String, String> getComponentToClusterInfoKeyMap() { - return componentToClusterInfoKeyMap; - } - public static long[] getRequestStage(String actionId) { String[] fields = actionId.split("-"); long[] requestStageIds = new long[2]; @@ -278,6 +242,16 @@ public class StageUtils { return commandParams; } + /** + * A helper method for generating keys for the clusterHostInfo section. + */ + public static String getClusterHostInfoKey(String componentName){ + if (componentName == null){ + throw new IllegalArgumentException("Component name cannot be null"); + } + return componentName.toLowerCase()+"_hosts"; + } + public static Map<String, Set<String>> getClusterHostInfo(Cluster cluster) throws AmbariException { //Fill hosts and ports lists Set<String> hostsSet = new LinkedHashSet<>(); @@ -325,21 +299,10 @@ public class StageUtils { ServiceComponent serviceComponent = serviceComponentEntry.getValue(); String componentName = serviceComponent.getName(); - String roleName = componentToClusterInfoKeyMap.get(componentName); - if(null == roleName) { - roleName = additionalComponentToClusterInfoKeyMap.get(componentName); - } - if (null == roleName && !serviceComponent.isClientComponent()) { - roleName = componentName.toLowerCase() + "_hosts"; - additionalComponentToClusterInfoKeyMap.put(componentName, roleName); - } + String roleName = getClusterHostInfoKey(componentName); String decomRoleName = decommissionedToClusterInfoKeyMap.get(componentName); - if (roleName == null && decomRoleName == null) { - continue; - } - for (String hostName : serviceComponent.getServiceComponentHosts().keySet()) { if (roleName != null) { @@ -380,7 +343,7 @@ public class StageUtils { Collection<String> hostComponents = entry.getValue(); for (String hostComponent : hostComponents) { - String roleName = componentToClusterInfoKeyMap.get(hostComponent); + String roleName = getClusterHostInfoKey(hostComponent); if (null == roleName) { roleName = additionalComponentToClusterInfoKeyMap.get(hostComponent); } diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java index 084efdd..af78e29 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java @@ -1318,7 +1318,7 @@ public class AmbariManagementControllerTest { assertTrue(ec.getCommandParams().containsKey("command_retry_enabled")); assertEquals("false", ec.getCommandParams().get("command_retry_enabled")); Map<String, Set<String>> chInfo = ec.getClusterHostInfo(); - assertTrue(chInfo.containsKey("namenode_host")); + assertTrue(chInfo.containsKey("namenode_hosts")); assertFalse(ec.getCommandParams().containsKey("custom_folder")); ec = controller.getExecutionCommand(cluster, diff --git a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java index 3e8d348..8f3fe8e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/utils/StageUtilsTest.java @@ -559,19 +559,19 @@ public class StageUtilsTest extends EasyMockSupport { assertEquals(StageUtils.getHostName(), serverHost.iterator().next()); // check host role replacing by the projected topology - assertTrue(getDecompressedSet(info.get("hbase_rs_hosts")).contains(9)); + assertTrue(getDecompressedSet(info.get("hbase_regionserver_hosts")).contains(9)); // Validate substitutions... info = StageUtils.substituteHostIndexes(info); - checkServiceHostNames(info, "DATANODE", "slave_hosts", projectedTopology); - checkServiceHostNames(info, "NAMENODE", "namenode_host", projectedTopology); - checkServiceHostNames(info, "SECONDARY_NAMENODE", "snamenode_host", projectedTopology); - checkServiceHostNames(info, "HBASE_MASTER", "hbase_master_hosts", projectedTopology); - checkServiceHostNames(info, "HBASE_REGIONSERVER", "hbase_rs_hosts", projectedTopology); - checkServiceHostNames(info, "JOBTRACKER", "jtnode_host", projectedTopology); - checkServiceHostNames(info, "TASKTRACKER", "mapred_tt_hosts", projectedTopology); - checkServiceHostNames(info, "NONAME_SERVER", "noname_server_hosts", projectedTopology); + checkServiceHostNames(info, "DATANODE", projectedTopology); + checkServiceHostNames(info, "NAMENODE", projectedTopology); + checkServiceHostNames(info, "SECONDARY_NAMENODE", projectedTopology); + checkServiceHostNames(info, "HBASE_MASTER", projectedTopology); + checkServiceHostNames(info, "HBASE_REGIONSERVER", projectedTopology); + checkServiceHostNames(info, "JOBTRACKER", projectedTopology); + checkServiceHostNames(info, "TASKTRACKER", projectedTopology); + checkServiceHostNames(info, "NONAME_SERVER", projectedTopology); } private void insertTopology(Map<String, Collection<String>> projectedTopology, String componentName, Set<String> hostNames) { @@ -700,7 +700,7 @@ public class StageUtilsTest extends EasyMockSupport { } // Determine the actual hosts for a given component... - Set<String> hosts = info.get(mappedComponentName); + Set<String> hosts = info.get(StageUtils.getClusterHostInfoKey(componentName)); if (hosts != null) { actualHostsList.addAll(getDecompressedSet(hosts)); } @@ -708,7 +708,7 @@ public class StageUtilsTest extends EasyMockSupport { assertEquals(expectedHostsList, actualHostsList); } - private void checkServiceHostNames(Map<String, Set<String>> info, String componentName, String mappedComponentName, + private void checkServiceHostNames(Map<String, Set<String>> info, String componentName, Map<String, Collection<String>> serviceTopology) { Set<String> expectedHostsList = new HashSet<>(); Set<String> actualHostsList = new HashSet<>(); @@ -721,7 +721,7 @@ public class StageUtilsTest extends EasyMockSupport { } // Determine the actual hosts for a given component... - Set<String> hosts = info.get(mappedComponentName); + Set<String> hosts = info.get(StageUtils.getClusterHostInfoKey(componentName)); if (hosts != null) { actualHostsList.addAll(hosts); } -- To stop receiving notification emails like this one, please contact swa...@apache.org.