Repository: ambari Updated Branches: refs/heads/trunk 2817ce67e -> 84214c5ae
AMBARI-15733 - Atlas Integration : Support Atlas HA Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/84214c5a Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/84214c5a Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/84214c5a Branch: refs/heads/trunk Commit: 84214c5ae1ba107931a735a685951239fc0c9a1e Parents: 2817ce6 Author: tbeerbower <tbeerbo...@hortonworks.com> Authored: Fri Apr 8 14:44:43 2016 -0400 Committer: tbeerbower <tbeerbo...@hortonworks.com> Committed: Fri Apr 8 14:44:43 2016 -0400 ---------------------------------------------------------------------- .../internal/AbstractProviderModule.java | 15 ++ .../AtlasServerHttpPropertyRequest.java | 72 ++++++ .../internal/HostComponentResourceProvider.java | 26 -- .../internal/HttpPropertyProvider.java | 212 +++++++++++++++ .../internal/HttpProxyPropertyProvider.java | 256 ------------------- .../internal/JsonHttpPropertyRequest.java | 97 +++++++ .../ResourceManagerHttpPropertyRequest.java | 122 +++++++++ .../ATLAS/0.1.0.2.3/package/scripts/params.py | 20 ++ .../configuration/application-properties.xml | 45 ++++ .../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 1 + .../AtlasServerHttpPropertyRequestTest.java | 81 ++++++ .../internal/HttpPropertyProviderTest.java | 160 ++++++++---- 12 files changed, 777 insertions(+), 330 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java index b77fda2..ca491f2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AbstractProviderModule.java @@ -93,6 +93,8 @@ public abstract class AbstractProviderModule implements ProviderModule, private static final Map<Service.Type, Map<String, String[]>> serviceDesiredProperties = new EnumMap<Service.Type, Map<String, String[]>>(Service.Type.class); private static final Map<String, Service.Type> componentServiceMap = new HashMap<String, Service.Type>(); + private static final Map<String, List<HttpPropertyProvider.HttpPropertyRequest>> HTTP_PROPERTY_REQUESTS = new HashMap<>(); + private static final String PROPERTY_HDFS_HTTP_POLICY_VALUE_HTTPS_ONLY = "HTTPS_ONLY"; private static final String COLLECTOR_DEFAULT_PORT = "6188"; @@ -171,6 +173,12 @@ public abstract class AbstractProviderModule implements ProviderModule, initPropMap.put("datanode", new String[]{"dfs.namenode.servicerpc-address.%s.%s"}); initPropMap.put("healthcheck", new String[]{"dfs.namenode.lifeline.rpc-address.%s.%s"}); jmxDesiredRpcSuffixProperties.put("NAMENODE-HA", initPropMap); + + HTTP_PROPERTY_REQUESTS.put("RESOURCEMANAGER", + Collections.<HttpPropertyProvider.HttpPropertyRequest>singletonList(new ResourceManagerHttpPropertyRequest())); + + HTTP_PROPERTY_REQUESTS.put("ATLAS_SERVER", + Collections.<HttpPropertyProvider.HttpPropertyRequest>singletonList(new AtlasServerHttpPropertyRequest())); } /** @@ -790,6 +798,13 @@ public abstract class AbstractProviderModule implements ProviderModule, PropertyHelper.getPropertyId("HostRoles", "state"), jpp, gpp)); + + providers.add(new HttpPropertyProvider(streamProvider, + managementController.getClusters(), + PropertyHelper.getPropertyId("HostRoles", "cluster_name"), + PropertyHelper.getPropertyId("HostRoles", "host_name"), + PropertyHelper.getPropertyId("HostRoles", "component_name"), + HTTP_PROPERTY_REQUESTS)); } break; case RootServiceComponent: http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AtlasServerHttpPropertyRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AtlasServerHttpPropertyRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AtlasServerHttpPropertyRequest.java new file mode 100644 index 0000000..e94e01b --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/AtlasServerHttpPropertyRequest.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.internal; + +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.state.Cluster; + +import java.util.Collections; +import java.util.Map; + +/** + * Atlas server specific HTTP property request. + */ +public class AtlasServerHttpPropertyRequest extends JsonHttpPropertyRequest { + + private static final String PROPERTY_ENABLE_TLS = "atlas.enableTLS"; + private static final String PROPERTY_SERVER_HTTPS_PORT = "atlas.server.https.port"; + private static final String PROPERTY_SERVER_HTTP_PORT = "atlas.server.http.port"; + private static final String CONFIG_APPLICATION_PROPERTIES = "application-properties"; + private static final String URL_TEMPLATE = "%s://%s:%s/api/atlas/admin/status"; + + private static final Map<String, String> PROPERTY_MAPPINGS = + Collections.singletonMap("Status", "HostRoles/ha_state"); + + + // ----- Constructors ---------------------------------------------------- + + public AtlasServerHttpPropertyRequest() { + super(PROPERTY_MAPPINGS); + } + + + // ----- PropertyRequest ------------------------------------------------- + + @Override + public String getUrl(Cluster cluster, String hostName) + throws SystemException { + + Map<String, String> atlasConfig = cluster.getDesiredConfigByType(CONFIG_APPLICATION_PROPERTIES).getProperties(); + + boolean useHttps = Boolean.parseBoolean(getConfigValue(atlasConfig, PROPERTY_ENABLE_TLS, "false")); + + String port = useHttps ? + getConfigValue(atlasConfig, PROPERTY_SERVER_HTTPS_PORT, "21443") : + getConfigValue(atlasConfig, PROPERTY_SERVER_HTTP_PORT, "21000"); + + return String.format(URL_TEMPLATE, useHttps ? "https" : "http", hostName, port); + } + + + // ----- helper methods -------------------------------------------------- + + // get a configuration property value + private String getConfigValue(Map<String, String> atlasConfig, String property, String defaultValue) { + return atlasConfig.containsKey(property) ? atlasConfig.get(property) : defaultValue; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java index 11db913..39a09a6 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HostComponentResourceProvider.java @@ -111,11 +111,6 @@ public class HostComponentResourceProvider extends AbstractControllerResourcePro = PropertyHelper.getPropertyId("HostRoles", "hdp_version"); public static final String HOST_COMPONENT_UPGRADE_STATE_PROPERTY_ID = "HostRoles/upgrade_state"; - //Component name mappings - private final Map<String, PropertyProvider> HOST_COMPONENT_PROPERTIES_PROVIDER = new HashMap<String, PropertyProvider>(); - private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_CONNECT_TIMEOUT = 1500; //milliseconds - private static final int HOST_COMPONENT_HTTP_PROPERTY_REQUEST_READ_TIMEOUT = 10000; //milliseconds - //Parameters from the predicate private static final String QUERY_PARAMETERS_RUN_SMOKE_TEST_ID = "params/run_smoke_test"; private static Set<String> pkPropertyIds = @@ -149,19 +144,6 @@ public class HostComponentResourceProvider extends AbstractControllerResourcePro @Assisted AmbariManagementController managementController, Injector injector) { super(propertyIds, keyPropertyIds, managementController); - ComponentSSLConfiguration configuration = ComponentSSLConfiguration.instance(); - URLStreamProvider streamProvider = new URLStreamProvider( - HOST_COMPONENT_HTTP_PROPERTY_REQUEST_CONNECT_TIMEOUT, - HOST_COMPONENT_HTTP_PROPERTY_REQUEST_READ_TIMEOUT, - configuration.getTruststorePath(), configuration.getTruststorePassword(), configuration.getTruststoreType()); - - HttpProxyPropertyProvider httpPropertyProvider = new HttpProxyPropertyProvider(streamProvider, - configuration, injector, - PropertyHelper.getPropertyId("HostRoles", "cluster_name"), - PropertyHelper.getPropertyId("HostRoles", "host_name"), - PropertyHelper.getPropertyId("HostRoles", "component_name")); - - HOST_COMPONENT_PROPERTIES_PROVIDER.put("RESOURCEMANAGER", httpPropertyProvider); setRequiredCreateAuthorizations(EnumSet.of(RoleAuthorization.SERVICE_ADD_DELETE_SERVICES,RoleAuthorization.HOST_ADD_DELETE_COMPONENTS)); setRequiredDeleteAuthorizations(EnumSet.of(RoleAuthorization.SERVICE_ADD_DELETE_SERVICES,RoleAuthorization.HOST_ADD_DELETE_COMPONENTS)); @@ -281,14 +263,6 @@ public class HostComponentResourceProvider extends AbstractControllerResourcePro response.getMaintenanceState(), requestedIds); } - String componentName = (String) resource.getPropertyValue(HOST_COMPONENT_COMPONENT_NAME_PROPERTY_ID); - PropertyProvider propertyProvider = HOST_COMPONENT_PROPERTIES_PROVIDER.get(componentName); - if (propertyProvider != null) { - Set<Resource> resourcesToPopulate = new HashSet<Resource>(); - resourcesToPopulate.add(resource); - propertyProvider.populateResources(resourcesToPopulate, request, predicate); - } - resources.add(resource); } return resources; http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java new file mode 100644 index 0000000..c53df3f --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpPropertyProvider.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.internal; + +import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.spi.Predicate; +import org.apache.ambari.server.controller.spi.PropertyProvider; +import org.apache.ambari.server.controller.spi.Request; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.controller.utilities.StreamProvider; +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Clusters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Property provider for host component resources that is used to read HTTP data from another server. + */ +public class HttpPropertyProvider extends BaseProvider implements PropertyProvider { + + protected final static Logger LOG = LoggerFactory.getLogger(HttpPropertyProvider.class); + + private final StreamProvider streamProvider; + private final String clusterNamePropertyId; + private final String hostNamePropertyId; + private final String componentNamePropertyId; + private final Clusters clusters; + private final Map<String, List<HttpPropertyRequest>> httpPropertyRequests; + + + // ----- Constructors ------------------------------------------------------ + + public HttpPropertyProvider( + StreamProvider stream, + Clusters clusters, + String clusterNamePropertyId, + String hostNamePropertyId, + String componentNamePropertyId, + Map<String, List<HttpPropertyRequest>> httpPropertyRequests) { + + super(getSupportedProperties(httpPropertyRequests)); + this.streamProvider = stream; + this.clusterNamePropertyId = clusterNamePropertyId; + this.hostNamePropertyId = hostNamePropertyId; + this.componentNamePropertyId = componentNamePropertyId; + this.clusters = clusters; + this.httpPropertyRequests = httpPropertyRequests; + } + + + // ----- PropertyProvider -------------------------------------------------- + + // get the complete set of Ambari properties that can be set by this property provider. + private static Set<String> getSupportedProperties(Map<String, List<HttpPropertyRequest>> httpPropertyRequests) { + Set<String> supportedProperties = new HashSet<>(); + + for (List<HttpPropertyRequest> httpPropertyRequestList : httpPropertyRequests.values()) { + for (HttpPropertyRequest httpPropertyRequest : httpPropertyRequestList) { + supportedProperties.addAll(httpPropertyRequest.getSupportedProperties()); + } + } + return Collections.unmodifiableSet(supportedProperties); + } + + + // ----- helper methods ---------------------------------------------------- + + @Override + public Set<Resource> populateResources(Set<Resource> resources, + Request request, Predicate predicate) throws SystemException { + + Set<String> ids = getRequestPropertyIds(request, predicate); + + if (ids.size() == 0) { + return resources; + } + + for (Resource resource : resources) { + String clusterName = (String) resource.getPropertyValue(clusterNamePropertyId); + String hostName = (String) resource.getPropertyValue(hostNamePropertyId); + String componentName = (String) resource.getPropertyValue(componentNamePropertyId); + + if (clusterName != null && hostName != null && componentName != null && + httpPropertyRequests.containsKey(componentName)) { + + try { + Cluster cluster = clusters.getCluster(clusterName); + + List<HttpPropertyRequest> httpPropertyRequestList = httpPropertyRequests.get(componentName); + + for (HttpPropertyRequest httpPropertyRequest : httpPropertyRequestList) { + populateResource(httpPropertyRequest, resource, cluster, hostName); + } + } catch (AmbariException e) { + String msg = String.format("Could not load cluster with name %s.", clusterName); + LOG.debug(msg, e); + throw new SystemException(msg, e); + } + } + } + return resources; + } + + // populate the given resource from the given HTTP property request. + private void populateResource(HttpPropertyRequest httpPropertyRequest, Resource resource, + Cluster cluster, String hostName) throws SystemException { + + String url = httpPropertyRequest.getUrl(cluster, hostName); + + try { + InputStream inputStream = streamProvider.readFrom(url); + + try { + httpPropertyRequest.populateResource(resource, inputStream); + } finally { + try { + inputStream.close(); + } catch (IOException ioe) { + LOG.error(String.format("Error closing HTTP response stream %s", url), ioe); + } + } + } catch (Exception e) { + LOG.debug(String.format("Error reading HTTP response from %s", url), e); + } + } + + + // ----- inner class : HttpPropertyRequest --------------------------------- + + /** + * Represents an HTTP request to another server for properties to be + * used to populate an Ambari resource. + */ + public static abstract class HttpPropertyRequest { + + private final Map<String, String> propertyMappings; + + + // ----- Constructors ---------------------------------------------------- + + protected HttpPropertyRequest(Map<String, String> propertyMappings) { + this.propertyMappings = propertyMappings; + } + + + // ----- PropertyRequest ------------------------------------------------- + + /** + * Get the names of the Ambari properties that can be set by this HTTP property request. + * + * @return the supported property names + */ + public Collection<String> getSupportedProperties() { + return propertyMappings.values(); + } + + /** + * Get the property name mappings from source property to Ambari property. + * + * @return the source to Ambari property name mappings + */ + protected Map<String, String> getPropertyMappings() { + return propertyMappings; + } + + /** + * Get the URL used to make the HTTP request. + * + * @param cluster the cluster of the resource being populated + * @param hostName the host name of the resource being populated + * @return the URL to make the HTTP request + * + * @throws SystemException if the URL can not be obtained + */ + public abstract String getUrl(Cluster cluster, String hostName) throws SystemException; + + /** + * Populate the given resource from the given input stream. + * + * @param resource the Ambari resource to populate + * @param inputStream the input stream from the HTTP request + * + * @throws SystemException if the resource can not be populated + */ + public abstract void populateResource(Resource resource, InputStream inputStream) throws SystemException; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpProxyPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpProxyPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpProxyPropertyProvider.java deleted file mode 100644 index e92536c..0000000 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/HttpProxyPropertyProvider.java +++ /dev/null @@ -1,256 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.ambari.server.controller.internal; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.reflect.Type; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import com.google.gson.JsonSyntaxException; -import com.google.inject.Injector; -import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.configuration.ComponentSSLConfiguration; -import org.apache.ambari.server.controller.spi.Predicate; -import org.apache.ambari.server.controller.spi.PropertyProvider; -import org.apache.ambari.server.controller.spi.Request; -import org.apache.ambari.server.controller.spi.Resource; -import org.apache.ambari.server.controller.spi.SystemException; -import org.apache.ambari.server.controller.utilities.PropertyHelper; -import org.apache.ambari.server.controller.utilities.StreamProvider; -import org.apache.ambari.server.state.Cluster; -import org.apache.ambari.server.state.Clusters; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; - -/** - * Property provider that is used to read HTTP data from another server. - */ -public class HttpProxyPropertyProvider extends BaseProvider implements PropertyProvider { - - protected final static Logger LOG = - LoggerFactory.getLogger(HttpProxyPropertyProvider.class); - - private static final Map<String, String> URL_TEMPLATES = new HashMap<String, String>(); - private static final Map<String, String> MAPPINGS = new HashMap<String, String>(); - private static final Map<String, String> PROPERTIES_TO_FILTER = new HashMap<String, String>(); - - private static final String COMPONENT_RESOURCEMANAGER = "RESOURCEMANAGER"; - private static final String CONFIG_YARN_SITE = "yarn-site"; - private static final String CONFIG_CORE_SITE = "core-site"; - private static final String PROPERTY_YARN_HTTP_POLICY = "yarn.http.policy"; - private static final String PROPERTY_HADOOP_SSL_ENABLED = "hadoop.ssl.enabled"; - private static final String PROPERTY_YARN_HTTP_POLICY_VALUE_HTTPS_ONLY = "HTTPS_ONLY"; - private static final String PROPERTY_HADOOP_SSL_ENABLED_VALUE_TRUE = "true"; - - - private static final String PROPERTY_RESOURCEMANAGER_WEBAPP_ADDRESS = "yarn.resourcemanager.webapp.address"; - private static final String PROPERTY_RESOURCEMANAGER_WEBAPP_HTTPS_ADDRESS = "yarn.resourcemanager.webapp.https.address"; - // resource manager HA properties - - private static final String PROPERTY_RESOURCEMANAGER_HA_RM_IDS = "yarn.resourcemanager.ha.rm-ids"; - private static final String PROPERTY_RESOURCEMANAGER_HOSTNAME_TEMPLATE = "yarn.resourcemanager.hostname.%s"; - private static final String PROPERTY_RESOURCEMANAGER_WEBAPP_ADDRESS_TEMPLATE = "yarn.resourcemanager.webapp.address.%s"; - private static final String PROPERTY_RESOURCEMANAGER_WEBAPP_HTTPS_ADDRESS_TEMPLATE = "yarn.resourcemanager.webapp.https.address.%s"; - - static { - URL_TEMPLATES.put(COMPONENT_RESOURCEMANAGER, "http://%s:%s/ws/v1/cluster/info";); - - MAPPINGS.put(COMPONENT_RESOURCEMANAGER, PropertyHelper.getPropertyId("HostRoles", "ha_state")); - - PROPERTIES_TO_FILTER.put(COMPONENT_RESOURCEMANAGER, "clusterInfo/haState"); - } - - private final ComponentSSLConfiguration configuration; - - private StreamProvider streamProvider = null; - // !!! not yet used, but make consistent - private String clusterNamePropertyId = null; - private String hostNamePropertyId = null; - private String componentNamePropertyId = null; - - private Injector injector; - private Clusters clusters; - - public HttpProxyPropertyProvider( - StreamProvider stream, - ComponentSSLConfiguration configuration, - Injector inject, - String clusterNamePropertyId, - String hostNamePropertyId, - String componentNamePropertyId) { - - super(new HashSet<String>(MAPPINGS.values())); - this.streamProvider = stream; - this.configuration = configuration; - this.clusterNamePropertyId = clusterNamePropertyId; - this.hostNamePropertyId = hostNamePropertyId; - this.componentNamePropertyId = componentNamePropertyId; - this.injector = inject; - this.clusters = injector.getInstance(Clusters.class); - } - - /** - * This method only checks if an HTTP-type property should be fulfilled. No - * modification is performed on the resources. - */ - @Override - public Set<Resource> populateResources(Set<Resource> resources, - Request request, Predicate predicate) throws SystemException { - - Set<String> ids = getRequestPropertyIds(request, predicate); - - if (0 == ids.size()) - return resources; - - for (Resource resource : resources) { - - Object hostName = resource.getPropertyValue(hostNamePropertyId); - Object componentName = resource.getPropertyValue(componentNamePropertyId); - Object clusterName = resource.getPropertyValue(clusterNamePropertyId); - - if (null != hostName && null != componentName && - MAPPINGS.containsKey(componentName.toString()) && - URL_TEMPLATES.containsKey(componentName.toString())) { - - String template = getTemplate(componentName.toString(), clusterName.toString(), hostName.toString()); - String propertyId = MAPPINGS.get(componentName.toString()); - String url = String.format(template, hostName); - - getHttpResponse(resource, url, propertyId); - } - } - - return resources; - } - - private String getTemplate(String componentName, String clusterName, String hostName) throws SystemException { - String template = URL_TEMPLATES.get(componentName); - - if (componentName.equals(COMPONENT_RESOURCEMANAGER)) { - try { - Cluster cluster = this.clusters.getCluster(clusterName); - Map<String, String> yarnConfigProperties = cluster.getDesiredConfigByType(CONFIG_YARN_SITE).getProperties(); - Map<String, String> coreConfigProperties = cluster.getDesiredConfigByType(CONFIG_CORE_SITE).getProperties(); - String yarnHttpPolicy = yarnConfigProperties.get(PROPERTY_YARN_HTTP_POLICY); - String hadoopSslEnabled = coreConfigProperties.get(PROPERTY_HADOOP_SSL_ENABLED); - boolean useHttps = (yarnHttpPolicy != null && yarnHttpPolicy.equals(PROPERTY_YARN_HTTP_POLICY_VALUE_HTTPS_ONLY)) || - hadoopSslEnabled != null && hadoopSslEnabled.equals(PROPERTY_HADOOP_SSL_ENABLED_VALUE_TRUE); - if (useHttps) { - template = template.replace("http", "https"); - } - // determine correct port, we don't want to use hardcoded one - String port; - if(!yarnConfigProperties.containsKey(PROPERTY_RESOURCEMANAGER_HA_RM_IDS)) { - //non ha mode - if(useHttps) { - port = getPortFromProperty(yarnConfigProperties, PROPERTY_RESOURCEMANAGER_WEBAPP_HTTPS_ADDRESS, "8090"); - } else { - port = getPortFromProperty(yarnConfigProperties, PROPERTY_RESOURCEMANAGER_WEBAPP_ADDRESS, "8088"); - } - } else { - // ha mode - String rmId = null; - for(String id : yarnConfigProperties.get(PROPERTY_RESOURCEMANAGER_HA_RM_IDS).split(",")) { - String hostNameProperty = String.format(PROPERTY_RESOURCEMANAGER_HOSTNAME_TEMPLATE, id); - String hostNameById = yarnConfigProperties.get(hostNameProperty); - if(hostNameById.equals(hostName)){ - rmId = id; - break; - } - } - if(useHttps) { - String httpsAddressPoperty = String.format(PROPERTY_RESOURCEMANAGER_WEBAPP_HTTPS_ADDRESS_TEMPLATE, rmId); - port = getPortFromProperty(yarnConfigProperties, httpsAddressPoperty, "8090"); - } else { - String httpAddressPoperty = String.format(PROPERTY_RESOURCEMANAGER_WEBAPP_ADDRESS_TEMPLATE, rmId); - port = getPortFromProperty(yarnConfigProperties, httpAddressPoperty, "8088"); - } - - } - template = String.format(template, "%s", port); - } catch (AmbariException e) { - LOG.debug(String.format("Could not load cluster with name %s. %s", clusterName, e.getMessage())); - throw new SystemException(String.format("Could not load cluster with name %s.", clusterName),e); - } - } - return template; - } - - private String getPortFromProperty(Map<String, String> propertyMap, String property, String defaultValue) { - if(propertyMap.containsKey(property)) - return propertyMap.get(property).split(":")[1]; - else - return defaultValue; - } - - private Object getPropertyValueToSet(Map<String, Object> propertyValueFromJson, Object componentName) throws SystemException { - Object result = propertyValueFromJson; - //TODO need refactoring for universalization - try { - if (PROPERTIES_TO_FILTER.get(componentName) != null) { - for (String key : PROPERTIES_TO_FILTER.get(componentName).split("/")) { - result = ((Map)result).get(key); - } - } - } catch (ClassCastException e) { - LOG.error(String.format("Error getting property value for %s. %s", PROPERTIES_TO_FILTER.get(componentName), - e.getMessage())); - throw new SystemException(String.format("Error getting property value for %s.", - PROPERTIES_TO_FILTER.get(componentName)),e); - } - return result; - } - - private void getHttpResponse(Resource r, String url, String propertyIdToSet) throws SystemException { - InputStream in = null; - try { - in = streamProvider.readFrom(url); - Type mapType = new TypeToken<Map<String, Object>>(){}.getType(); - Map<String, Object> propertyValueFromJson = new Gson().fromJson(IOUtils.toString(in, "UTF-8"), mapType); - Object propertyValueToSet = getPropertyValueToSet(propertyValueFromJson, - r.getPropertyValue(componentNamePropertyId)); - r.setProperty(propertyIdToSet, propertyValueToSet); - } - catch (IOException ioe) { - LOG.debug("Error reading HTTP response from " + url); - r.setProperty(propertyIdToSet, null); - } catch (JsonSyntaxException jse) { - LOG.error("Error parsing HTTP response from " + url); - r.setProperty(propertyIdToSet, null); - } finally { - if (in != null) { - try { - in.close(); - } - catch (IOException ioe) { - LOG.error("Error closing HTTP response stream " + url); - } - } - } - - } - -} http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/JsonHttpPropertyRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/JsonHttpPropertyRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/JsonHttpPropertyRequest.java new file mode 100644 index 0000000..6a93b4b --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/JsonHttpPropertyRequest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.internal; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.apache.ambari.server.controller.spi.Resource; +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.lang.reflect.Type; +import java.util.Map; + +/** + * Represents an HTTP request to another server for properties to be used to populate an Ambari resource. + * The server response is expected to be JSON that can be deserialized into a <code>Map<String, Object>></code> + * instance. + */ +public abstract class JsonHttpPropertyRequest extends HttpPropertyProvider.HttpPropertyRequest { + protected final static Logger LOG = LoggerFactory.getLogger(JsonHttpPropertyRequest.class); + + private static final Type MAP_TYPE = new TypeToken<Map<String, Object>>() {}.getType(); + + private static final Gson GSON = new Gson(); + + + // ----- Constructors ---------------------------------------------------- + + /** + * Create a property request. + * + * @param propertyMappings the property name mapping + */ + public JsonHttpPropertyRequest(Map<String, String> propertyMappings) { + super(propertyMappings); + } + + + // ----- PropertyRequest ------------------------------------------------- + + @Override + public void populateResource(Resource resource, InputStream inputStream) throws SystemException { + + try { + Map<String, Object> responseMap = GSON.fromJson(IOUtils.toString(inputStream, "UTF-8"), MAP_TYPE); + + for (Map.Entry<String, String> entry : getPropertyMappings().entrySet()) { + Object propertyValueToSet = getPropertyValue(responseMap, entry.getKey()); + resource.setProperty(entry.getValue(), propertyValueToSet); + } + } catch (IOException e) { + throw new SystemException("Error setting properties.", e); + } + } + + + // ----- helper methods -------------------------------------------------- + + // get the property value from the response map for the given property name + private Object getPropertyValue(Map<String, Object> responseMap, String property) throws SystemException { + if (property == null) { + return null; + } + + Object result = responseMap; + + try { + for (String key : property.split("/")) { + result = ((Map) result).get(key); + } + } catch (ClassCastException e) { + String msg = String.format("Error getting property value for %s.", property); + LOG.error(msg, e); + throw new SystemException(msg, e); + } + return result; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ResourceManagerHttpPropertyRequest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ResourceManagerHttpPropertyRequest.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ResourceManagerHttpPropertyRequest.java new file mode 100644 index 0000000..a74622a --- /dev/null +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ResourceManagerHttpPropertyRequest.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.internal; + +import org.apache.ambari.server.controller.spi.SystemException; +import org.apache.ambari.server.state.Cluster; + +import java.util.Collections; +import java.util.Map; + +/** + * Resource manager specific HTTP property request. + */ +public class ResourceManagerHttpPropertyRequest extends JsonHttpPropertyRequest { + + private static final String CONFIG_YARN_SITE = "yarn-site"; + private static final String CONFIG_CORE_SITE = "core-site"; + + private static final String PROPERTY_YARN_HTTP_POLICY = "yarn.http.policy"; + private static final String PROPERTY_HADOOP_SSL_ENABLED = "hadoop.ssl.enabled"; + private static final String PROPERTY_YARN_HTTP_POLICY_VALUE_HTTPS_ONLY = "HTTPS_ONLY"; + private static final String PROPERTY_HADOOP_SSL_ENABLED_VALUE_TRUE = "true"; + + // resource manager properties + private static final String PROPERTY_WEBAPP_ADDRESS = "yarn.resourcemanager.webapp.address"; + private static final String PROPERTY_WEBAPP_HTTPS_ADDRESS = "yarn.resourcemanager.webapp.https.address"; + + // resource manager HA properties + private static final String PROPERTY_HA_RM_IDS = "yarn.resourcemanager.ha.rm-ids"; + private static final String PROPERTY_HOSTNAME_TEMPLATE = "yarn.resourcemanager.hostname.%s"; + private static final String PROPERTY_WEBAPP_ADDRESS_TEMPLATE = "yarn.resourcemanager.webapp.address.%s"; + private static final String PROPERTY_WEBAPP_HTTPS_ADDRESS_TEMPLATE = "yarn.resourcemanager.webapp.https.address.%s"; + + private static final String URL_TEMPLATE = "%s://%s:%s/ws/v1/cluster/info"; + + private static final Map<String, String> PROPERTY_MAPPINGS = + Collections.singletonMap("clusterInfo/haState", "HostRoles/ha_state"); + + + // ----- Constructors ---------------------------------------------------- + + public ResourceManagerHttpPropertyRequest() { + super(PROPERTY_MAPPINGS); + } + + + // ----- PropertyRequest ------------------------------------------------- + + @Override + public String getUrl(Cluster cluster, String hostName) + throws SystemException { + Map<String, String> yarnConfig = cluster.getDesiredConfigByType(CONFIG_YARN_SITE).getProperties(); + Map<String, String> coreConfig = cluster.getDesiredConfigByType(CONFIG_CORE_SITE).getProperties(); + + String yarnHttpPolicy = yarnConfig.get(PROPERTY_YARN_HTTP_POLICY); + String hadoopSslEnabled = coreConfig.get(PROPERTY_HADOOP_SSL_ENABLED); + + boolean useHttps = + (yarnHttpPolicy != null && yarnHttpPolicy.equals(PROPERTY_YARN_HTTP_POLICY_VALUE_HTTPS_ONLY)) || + (hadoopSslEnabled != null && hadoopSslEnabled.equals(PROPERTY_HADOOP_SSL_ENABLED_VALUE_TRUE)); + + return String.format(URL_TEMPLATE, getProtocol(useHttps), hostName, getPort(hostName, yarnConfig, useHttps)); + } + + + // ----- helper methods -------------------------------------------------- + + // get the protocal - http or https + private String getProtocol(boolean useHttps) { + return useHttps ? "https" : "http"; + } + + // get the port for the HTTP request from the given config + private String getPort(String hostName, Map<String, String> yarnConfig, boolean useHttps) { + if (yarnConfig.containsKey(PROPERTY_HA_RM_IDS)) { + // ha mode + String rmId = getConfigResourceManagerId(yarnConfig, hostName); + return useHttps ? + getConfigPortValue(yarnConfig, String.format(PROPERTY_WEBAPP_HTTPS_ADDRESS_TEMPLATE, rmId), "8090") : + getConfigPortValue(yarnConfig, String.format(PROPERTY_WEBAPP_ADDRESS_TEMPLATE, rmId), "8088"); + } + //non ha mode + return useHttps ? + getConfigPortValue(yarnConfig, PROPERTY_WEBAPP_HTTPS_ADDRESS, "8090") : + getConfigPortValue(yarnConfig, PROPERTY_WEBAPP_ADDRESS, "8088"); + } + + // get the resource manager id from the given config + private String getConfigResourceManagerId(Map<String, String> yarnConfig, String hostName) { + + for (String id : yarnConfig.get(PROPERTY_HA_RM_IDS).split(",")) { + + String hostNameProperty = String.format(PROPERTY_HOSTNAME_TEMPLATE, id); + String hostNameById = yarnConfig.get(hostNameProperty); + + if (hostNameById.equals(hostName)) { + return id; + } + } + return null; + } + + // get the port for the HTTP request by splitting off of the property specifed from the given config + private String getConfigPortValue(Map<String, String> yarnConfig, String property, String defaultValue) { + return yarnConfig.containsKey(property) ? yarnConfig.get(property).split(":")[1] : defaultValue; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py index b377757..8c7872a 100644 --- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py @@ -127,3 +127,23 @@ if not len(kafka_broker_hosts) == 0: kafka_bootstrap_servers = kafka_broker_hosts[0] + ":" + str(kafka_broker_port) kafka_zookeeper_connect = default("/configurations/kafka-broker/zookeeper.connect", None) + +# atlas HA +atlas_hosts = sorted(default('/clusterHostInfo/atlas_server_hosts', [])) + +id = 1 +server_ids = "" +server_hosts = "" +first_id = True +for host in atlas_hosts: + server_id = "id" + str(id) + server_host = host + ":" + metadata_port + if first_id: + server_ids = server_id + server_hosts = server_host + else: + server_ids += "," + server_id + server_hosts += "\n" + "atlas.server.host." + server_id + "=" + server_host + + id += 1 + first_id = False http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml new file mode 100644 index 0000000..09cb987 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/application-properties.xml @@ -0,0 +1,45 @@ +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<!-- +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +--> +<configuration supports_final="false"> + <property> + <name>atlas.server.ha.enabled</name> + <value>false</value> + <description>Atlas high availability feature toggle.</description> + </property> + <property> + <name>atlas.server.ids</name> + <value>{{server_ids}}</value> + <description>List of Atlas server ids for HA feature.</description> + <value-attributes> + <overridable>false</overridable> + <visible>false</visible> + </value-attributes> + </property> + <property> + <name>atlas.server.host.id1</name> + <value>{{server_hosts}}</value> + <description>Mapping of Atlas server ids to hosts.</description> + <value-attributes> + <overridable>false</overridable> + <visible>false</visible> + </value-attributes> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml index 7061d6b..a25c2ae 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml @@ -24,6 +24,7 @@ <components> <component> <name>ATLAS_SERVER</name> + <cardinality>1+</cardinality> <configFile> <type>properties</type> <fileName>atlas-application.properties</fileName> http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AtlasServerHttpPropertyRequestTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AtlasServerHttpPropertyRequestTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AtlasServerHttpPropertyRequestTest.java new file mode 100644 index 0000000..70a6a9d --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/AtlasServerHttpPropertyRequestTest.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ambari.server.controller.internal; + +import org.apache.ambari.server.state.Cluster; +import org.apache.ambari.server.state.Config; +import org.junit.Assert; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.easymock.EasyMock.createNiceMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; + +/** + * AtlasServerHttpPropertyRequest tests. + */ +public class AtlasServerHttpPropertyRequestTest { + + @Test + public void testGetUrl() throws Exception { + + Cluster cluster = createNiceMock(Cluster.class); + Config config = createNiceMock(Config.class); + + Map<String, String> map = new HashMap<>(); + map.put("atlas.enableTLS", "false"); + map.put("atlas.server.http.port", "21000"); + + expect(cluster.getDesiredConfigByType("application-properties")).andReturn(config).anyTimes(); + expect(config.getProperties()).andReturn(map).anyTimes(); + + replay(cluster, config); + + AtlasServerHttpPropertyRequest propertyRequest = new AtlasServerHttpPropertyRequest(); + + String url = propertyRequest.getUrl(cluster, "host1"); + + Assert.assertEquals("http://host1:21000/api/atlas/admin/status";, url); + } + + @Test + public void testGetUrl_https() throws Exception { + + Cluster cluster = createNiceMock(Cluster.class); + Config config = createNiceMock(Config.class); + + Map<String, String> map = new HashMap<>(); + map.put("atlas.enableTLS", "true"); + map.put("atlas.server.https.port", "21443"); + + expect(cluster.getDesiredConfigByType("application-properties")).andReturn(config).anyTimes(); + expect(config.getProperties()).andReturn(map).anyTimes(); + + replay(cluster, config); + + AtlasServerHttpPropertyRequest propertyRequest = new AtlasServerHttpPropertyRequest(); + + String url = propertyRequest.getUrl(cluster, "host1"); + + Assert.assertEquals("https://host1:21443/api/atlas/admin/status";, url); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/84214c5a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java index b622728..c4fcca0 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/internal/HttpPropertyProviderTest.java @@ -25,15 +25,12 @@ import static org.easymock.EasyMock.replay; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.ambari.server.configuration.ComponentSSLConfiguration; -import org.apache.ambari.server.configuration.ComponentSSLConfigurationTest; import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.utilities.PropertyHelper; @@ -43,12 +40,7 @@ import org.apache.ambari.server.state.Clusters; import org.apache.ambari.server.state.Config; import org.junit.Assert; import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import com.google.inject.Injector; - -@RunWith(Parameterized.class) public class HttpPropertyProviderTest { private static final String PROPERTY_ID_CLUSTER_NAME = PropertyHelper.getPropertyId("HostRoles", "cluster_name"); private static final String PROPERTY_ID_HOST_NAME = PropertyHelper.getPropertyId("HostRoles", "host_name"); @@ -57,26 +49,14 @@ public class HttpPropertyProviderTest { private static final String PROPERTY_ID_STALE_CONFIGS = PropertyHelper.getPropertyId( "HostRoles", "stale_configs"); - private ComponentSSLConfiguration configuration; - - @Parameterized.Parameters - public static Collection<Object[]> configs() { - ComponentSSLConfiguration configuration1 = ComponentSSLConfigurationTest.getConfiguration( - "tspath", "tspass", "tstype", false); - - ComponentSSLConfiguration configuration2 = ComponentSSLConfigurationTest.getConfiguration( - "tspath", "tspass", "tstype", true); - - ComponentSSLConfiguration configuration3 = ComponentSSLConfigurationTest.getConfiguration( - "tspath", "tspass", "tstype", false); - - return Arrays.asList(new Object[][] { { configuration1 }, - { configuration2 }, { configuration3 } }); - } + private static final Map<String, List<HttpPropertyProvider.HttpPropertyRequest>> HTTP_PROPERTY_REQUESTS = new HashMap<>(); + static { + HTTP_PROPERTY_REQUESTS.put("RESOURCEMANAGER", + Collections.<HttpPropertyProvider.HttpPropertyRequest>singletonList(new ResourceManagerHttpPropertyRequest())); - public HttpPropertyProviderTest(ComponentSSLConfiguration configuration) { - this.configuration = configuration; + HTTP_PROPERTY_REQUESTS.put("ATLAS_SERVER", + Collections.<HttpPropertyProvider.HttpPropertyRequest>singletonList(new AtlasServerHttpPropertyRequest())); } @Test @@ -85,29 +65,28 @@ public class HttpPropertyProviderTest { TestStreamProvider streamProvider = new TestStreamProvider(false); Clusters clusters = createNiceMock(Clusters.class); Cluster cluster = createNiceMock(Cluster.class); - Injector injector = createNiceMock(Injector.class); Config config1 = createNiceMock(Config.class); Config config2 = createNiceMock(Config.class); - Map<String, String> map = new HashMap<String, String>(); + Map<String, String> map = new HashMap<>(); map.put("yarn.http.policy", "HTTPS_ONLY"); map.put("yarn.resourcemanager.webapp.https.address", "ec2-54-234-33-50.compute-1.amazonaws.com:8999"); map.put("yarn.resourcemanager.webapp.address", "ec2-54-234-33-50.compute-1.amazonaws.com:8088"); - expect(injector.getInstance(Clusters.class)).andReturn(clusters); expect(clusters.getCluster("testCluster")).andReturn(cluster); expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(config1).anyTimes(); expect(cluster.getDesiredConfigByType("core-site")).andReturn(config2).anyTimes(); expect(config1.getProperties()).andReturn(map).anyTimes(); expect(config2.getProperties()).andReturn(new HashMap<String, String>()).anyTimes(); - replay(injector, clusters, cluster, config1, config2); + replay(clusters, cluster, config1, config2); - HttpProxyPropertyProvider propProvider = new HttpProxyPropertyProvider( - streamProvider, configuration, injector, + HttpPropertyProvider propProvider = new HttpPropertyProvider( + streamProvider, clusters, PROPERTY_ID_CLUSTER_NAME, PROPERTY_ID_HOST_NAME, - PROPERTY_ID_COMPONENT_NAME); + PROPERTY_ID_COMPONENT_NAME, + HTTP_PROPERTY_REQUESTS); Resource resource = new ResourceImpl(Resource.Type.HostComponent); @@ -130,11 +109,10 @@ public class HttpPropertyProviderTest { TestStreamProvider streamProvider = new TestStreamProvider(false); Clusters clusters = createNiceMock(Clusters.class); Cluster cluster = createNiceMock(Cluster.class); - Injector injector = createNiceMock(Injector.class); Config config1 = createNiceMock(Config.class); Config config2 = createNiceMock(Config.class); - Map<String, String> map = new HashMap<String, String>(); + Map<String, String> map = new HashMap<>(); map.put("yarn.http.policy", "HTTPS_ONLY"); map.put("yarn.resourcemanager.ha.rm-ids", "rm1,rm2"); map.put("yarn.resourcemanager.hostname.rm1", "lc6402.ambari.apache.org"); @@ -144,20 +122,20 @@ public class HttpPropertyProviderTest { map.put("yarn.resourcemanager.webapp.https.address.rm1", "lc6402.ambari.apache.org:8066"); map.put("yarn.resourcemanager.webapp.https.address.rm2", "lc6403.ambari.apache.org:8066"); - expect(injector.getInstance(Clusters.class)).andReturn(clusters); expect(clusters.getCluster("testCluster")).andReturn(cluster); expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(config1).anyTimes(); expect(cluster.getDesiredConfigByType("core-site")).andReturn(config2).anyTimes(); expect(config1.getProperties()).andReturn(map).anyTimes(); expect(config2.getProperties()).andReturn(new HashMap<String, String>()).anyTimes(); - replay(injector, clusters, cluster, config1, config2); + replay(clusters, cluster, config1, config2); - HttpProxyPropertyProvider propProvider = new HttpProxyPropertyProvider( - streamProvider, configuration, injector, + HttpPropertyProvider propProvider = new HttpPropertyProvider( + streamProvider, clusters, PROPERTY_ID_CLUSTER_NAME, PROPERTY_ID_HOST_NAME, - PROPERTY_ID_COMPONENT_NAME); + PROPERTY_ID_COMPONENT_NAME, + HTTP_PROPERTY_REQUESTS); Resource resource = new ResourceImpl(Resource.Type.HostComponent); @@ -174,6 +152,85 @@ public class HttpPropertyProviderTest { "/ws/v1/cluster/info")); } + @Test + public void testPopulateResources_atlasServer() throws Exception { + + TestStreamProvider streamProvider = new TestStreamProvider("{\"Status\":\"ACTIVE\"}", false); + Clusters clusters = createNiceMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + Config config1 = createNiceMock(Config.class); + + Map<String, String> map = new HashMap<>(); + map.put("atlas.enableTLS", "false"); + map.put("atlas.server.http.port", "21000"); + + expect(clusters.getCluster("testCluster")).andReturn(cluster); + expect(cluster.getDesiredConfigByType("application-properties")).andReturn(config1).anyTimes(); + expect(config1.getProperties()).andReturn(map).anyTimes(); + + replay(clusters, cluster, config1); + + HttpPropertyProvider propProvider = new HttpPropertyProvider( + streamProvider, clusters, + PROPERTY_ID_CLUSTER_NAME, + PROPERTY_ID_HOST_NAME, + PROPERTY_ID_COMPONENT_NAME, + HTTP_PROPERTY_REQUESTS); + + Resource resource = new ResourceImpl(Resource.Type.HostComponent); + + resource.setProperty(PROPERTY_ID_CLUSTER_NAME, "testCluster"); + resource.setProperty(PROPERTY_ID_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com"); + resource.setProperty(PROPERTY_ID_COMPONENT_NAME, "ATLAS_SERVER"); + + Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet()); + + propProvider.populateResources(Collections.singleton(resource), request, null); + + Assert.assertEquals("ACTIVE", resource.getPropertiesMap().get("HostRoles").get("ha_state")); + Assert.assertEquals("http://ec2-54-234-33-50.compute-1.amazonaws.com:21000/api/atlas/admin/status";, + streamProvider.getLastSpec()); + } + + @Test + public void testPopulateResources_atlasServer_https() throws Exception { + + TestStreamProvider streamProvider = new TestStreamProvider("{\"Status\":\"ACTIVE\"}", false); + Clusters clusters = createNiceMock(Clusters.class); + Cluster cluster = createNiceMock(Cluster.class); + Config config1 = createNiceMock(Config.class); + + Map<String, String> map = new HashMap<>(); + map.put("atlas.enableTLS", "true"); + map.put("atlas.server.https.port", "21443"); + + expect(clusters.getCluster("testCluster")).andReturn(cluster); + expect(cluster.getDesiredConfigByType("application-properties")).andReturn(config1).anyTimes(); + expect(config1.getProperties()).andReturn(map).anyTimes(); + + replay(clusters, cluster, config1); + + HttpPropertyProvider propProvider = new HttpPropertyProvider( + streamProvider, clusters, + PROPERTY_ID_CLUSTER_NAME, + PROPERTY_ID_HOST_NAME, + PROPERTY_ID_COMPONENT_NAME, + HTTP_PROPERTY_REQUESTS); + + Resource resource = new ResourceImpl(Resource.Type.HostComponent); + + resource.setProperty(PROPERTY_ID_CLUSTER_NAME, "testCluster"); + resource.setProperty(PROPERTY_ID_HOST_NAME, "ec2-54-234-33-50.compute-1.amazonaws.com"); + resource.setProperty(PROPERTY_ID_COMPONENT_NAME, "ATLAS_SERVER"); + + Request request = PropertyHelper.getReadRequest(Collections.<String>emptySet()); + + propProvider.populateResources(Collections.singleton(resource), request, null); + + Assert.assertEquals("ACTIVE", resource.getPropertiesMap().get("HostRoles").get("ha_state")); + Assert.assertEquals("https://ec2-54-234-33-50.compute-1.amazonaws.com:21443/api/atlas/admin/status";, + streamProvider.getLastSpec()); + } @Test public void testReadGangliaServer() throws Exception { @@ -186,13 +243,14 @@ public class HttpPropertyProviderTest { private Resource doPopulate(String componentName, Set<String> requestProperties, StreamProvider streamProvider) throws Exception { - Injector injector = createNiceMock(Injector.class); + Clusters clusters = createNiceMock(Clusters.class); - HttpProxyPropertyProvider propProvider = new HttpProxyPropertyProvider( - streamProvider, configuration, injector, + HttpPropertyProvider propProvider = new HttpPropertyProvider( + streamProvider, clusters, PROPERTY_ID_CLUSTER_NAME, PROPERTY_ID_HOST_NAME, - PROPERTY_ID_COMPONENT_NAME); + PROPERTY_ID_COMPONENT_NAME, + HTTP_PROPERTY_REQUESTS); Resource resource = new ResourceImpl(Resource.Type.HostComponent); @@ -211,11 +269,18 @@ public class HttpPropertyProviderTest { private boolean throwError = false; private String lastSpec = null; private boolean isLastSpecUpdated; + private String responseStr = "{\"alerts\": [{\"Alert Body\": \"Body\"}],\"clusterInfo\": {\"haState\": \"ACTIVE\"}," + + " \"hostcounts\": {\"up_hosts\":\"1\", \"down_hosts\":\"0\"}}"; private TestStreamProvider(boolean throwErr) { throwError = throwErr; } + private TestStreamProvider(String responseStr, boolean throwErr) { + this.responseStr = responseStr; + throwError = throwErr; + } + @Override public InputStream readFrom(String spec) throws IOException { if (!isLastSpecUpdated) { @@ -228,9 +293,8 @@ public class HttpPropertyProviderTest { throw new IOException("Fake error"); } - String responseStr = "{\"alerts\": [{\"Alert Body\": \"Body\"}],\"clusterInfo\": {\"haState\": \"ACTIVE\"}," - + " \"hostcounts\": {\"up_hosts\":\"1\", \"down_hosts\":\"0\"}}"; - return new ByteArrayInputStream(responseStr.getBytes("UTF-8")); + + return new ByteArrayInputStream(responseStr.getBytes("UTF-8")); } public String getLastSpec() {
