KNOX-1014 - Service Discovery and Topology Generation Framework (Phil Zampino via lmccay)
Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/c2ca4432 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/c2ca4432 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/c2ca4432 Branch: refs/heads/master Commit: c2ca443262a848ae0d56e03e92ecba32fbf149f2 Parents: 3a0119b Author: Larry McCay <lmc...@hortonworks.com> Authored: Sat Sep 23 11:04:39 2017 -0400 Committer: Larry McCay <lmc...@hortonworks.com> Committed: Sat Sep 23 11:04:39 2017 -0400 ---------------------------------------------------------------------- b/gateway-discovery-ambari/pom.xml | 66 ++ .../discovery/ambari/AmbariCluster.java | 114 +++ .../discovery/ambari/AmbariComponent.java | 76 ++ .../ambari/AmbariServiceDiscovery.java | 291 +++++++ .../ambari/AmbariServiceDiscoveryMessages.java | 81 ++ .../ambari/AmbariServiceDiscoveryType.java | 35 + .../ambari/AmbariServiceURLCreator.java | 184 ++++ ...eway.topology.discovery.ServiceDiscoveryType | 19 + .../ambari/AmbariServiceDiscoveryTest.java | 856 +++++++++++++++++++ b/gateway-release/home/conf/descriptors/README | 1 + .../home/conf/shared-providers/README | 1 + gateway-discovery-ambari/pom.xml | 66 ++ .../discovery/ambari/AmbariCluster.java | 114 +++ .../discovery/ambari/AmbariComponent.java | 76 ++ .../ambari/AmbariServiceDiscovery.java | 291 +++++++ .../ambari/AmbariServiceDiscoveryMessages.java | 81 ++ .../ambari/AmbariServiceDiscoveryType.java | 35 + .../ambari/AmbariServiceURLCreator.java | 184 ++++ ...eway.topology.discovery.ServiceDiscoveryType | 19 + .../ambari/AmbariServiceDiscoveryTest.java | 856 +++++++++++++++++++ gateway-release/home/conf/descriptors/README | 1 + .../home/conf/shared-providers/README | 1 + gateway-release/pom.xml | 4 + gateway-server/pom.xml | 5 + .../apache/hadoop/gateway/GatewayMessages.java | 9 +- .../services/DefaultGatewayServices.java | 3 +- .../topology/impl/DefaultTopologyService.java | 278 +++++- .../builder/BeanPropertyTopologyBuilder.java | 2 +- .../DefaultServiceDiscoveryConfig.java | 48 ++ .../discovery/ServiceDiscoveryFactory.java | 81 ++ .../topology/simple/SimpleDescriptor.java | 46 + .../simple/SimpleDescriptorFactory.java | 71 ++ .../simple/SimpleDescriptorHandler.java | 186 ++++ .../topology/simple/SimpleDescriptorImpl.java | 111 +++ .../simple/SimpleDescriptorMessages.java | 44 + .../topology/DefaultTopologyServiceTest.java | 70 +- .../PropertiesFileServiceDiscoveryTest.java | 90 ++ .../discovery/ServiceDiscoveryFactoryTest.java | 81 ++ .../test/extension/DummyServiceDiscovery.java | 66 ++ .../extension/DummyServiceDiscoveryType.java | 32 + .../PropertiesFileServiceDiscovery.java | 108 +++ .../PropertiesFileServiceDiscoveryType.java | 35 + .../extension/SneakyServiceDiscoveryImpl.java | 40 + .../extension/SneakyServiceDiscoveryType.java | 33 + .../simple/SimpleDescriptorFactoryTest.java | 218 +++++ .../simple/SimpleDescriptorHandlerTest.java | 239 ++++++ ...eway.topology.discovery.ServiceDiscoveryType | 21 + .../topology/file/ambari-cluster-policy.xml | 74 ++ .../topology/file/simple-topology-four.json | 18 + .../topology/discovery/GatewayService.java | 29 + .../topology/discovery/ServiceDiscovery.java | 76 ++ .../discovery/ServiceDiscoveryConfig.java | 42 + .../discovery/ServiceDiscoveryType.java | 40 + pom.xml | 27 +- 54 files changed, 5626 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/pom.xml ---------------------------------------------------------------------- diff --git a/b/gateway-discovery-ambari/pom.xml b/b/gateway-discovery-ambari/pom.xml new file mode 100644 index 0000000..924e89c --- /dev/null +++ b/b/gateway-discovery-ambari/pom.xml @@ -0,0 +1,66 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.knox</groupId> + <artifactId>gateway</artifactId> + <version>0.14.0-SNAPSHOT</version> + </parent> + <artifactId>gateway-discovery-ambari</artifactId> + + <name>gateway-discovery-ambari</name> + <description>The extension to the gateway for service discovery using Apache Ambari.</description> + + <licenses> + <license> + <name>The Apache Software License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + <distribution>repo</distribution> + </license> + </licenses> + + <dependencies> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-spi</artifactId> + </dependency> + + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-test-utils</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <scope>test</scope> + </dependency> + + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java ---------------------------------------------------------------------- diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java new file mode 100644 index 0000000..6eaabd3 --- /dev/null +++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariCluster.java @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.discovery.ambari; + +import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class AmbariCluster implements ServiceDiscovery.Cluster { + + private String name = null; + + private AmbariServiceURLCreator urlCreator = new AmbariServiceURLCreator(); + + private Map<String, Map<String, ServiceConfiguration>> serviceConfigurations = new HashMap<>(); + + private Map<String, AmbariComponent> components = null; + + + AmbariCluster(String name) { + this.name = name; + components = new HashMap<String, AmbariComponent>(); + } + + void addServiceConfiguration(String serviceName, String configurationType, ServiceConfiguration serviceConfig) { + if (!serviceConfigurations.keySet().contains(serviceName)) { + serviceConfigurations.put(serviceName, new HashMap<String, ServiceConfiguration>()); + } + serviceConfigurations.get(serviceName).put(configurationType, serviceConfig); + } + + + void addComponent(AmbariComponent component) { + components.put(component.getName(), component); + } + + + ServiceConfiguration getServiceConfiguration(String serviceName, String configurationType) { + ServiceConfiguration sc = null; + Map<String, ServiceConfiguration> configs = serviceConfigurations.get(serviceName); + if (configs != null) { + sc = configs.get(configurationType); + } + return sc; + } + + + Map<String, AmbariComponent> getComponents() { + return components; + } + + + AmbariComponent getComponent(String name) { + return components.get(name); + } + + + @Override + public String getName() { + return name; + } + + + @Override + public List<String> getServiceURLs(String serviceName) { + List<String> urls = new ArrayList<>(); + urls.addAll(urlCreator.create(this, serviceName)); + return urls; + } + + + static class ServiceConfiguration { + + private String type; + private String version; + private Map<String, String> props; + + ServiceConfiguration(String type, String version, Map<String, String> properties) { + this.type = type; + this.version = version; + this.props = properties; + } + + public String getVersion() { + return version; + } + + public String getType() { + return type; + } + + public Map<String, String> getProperties() { + return props; + } + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java ---------------------------------------------------------------------- diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java new file mode 100644 index 0000000..55257fb --- /dev/null +++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariComponent.java @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.gateway.topology.discovery.ambari; + +import java.util.List; +import java.util.Map; + +class AmbariComponent { + + private String clusterName = null; + private String serviceName = null; + private String name = null; + private String version = null; + + private List<String> hostNames = null; + + private Map<String, String> properties = null; + + AmbariComponent(String name, + String version, + String cluster, + String service, + List<String> hostNames, + Map<String, String> properties) { + this.name = name; + this.serviceName = service; + this.clusterName = cluster; + this.version = version; + this.hostNames = hostNames; + this.properties = properties; + } + + public String getVersion() { + return version; + } + + public String getName() { + return name; + } + + public String getServiceName() { + return serviceName; + } + + public String getClusterName() { + return clusterName; + } + + public List<String> getHostNames() { + return hostNames; + } + + public Map<String, String> getConfigProperties() { + return properties; + } + + public String getConfigProperty(String propertyName) { + return properties.get(propertyName); + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java ---------------------------------------------------------------------- diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java new file mode 100644 index 0000000..34f20a7 --- /dev/null +++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscovery.java @@ -0,0 +1,291 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.discovery.ambari; + +import net.minidev.json.JSONArray; +import net.minidev.json.JSONObject; +import net.minidev.json.JSONValue; +import org.apache.hadoop.gateway.config.ConfigurationException; +import org.apache.hadoop.gateway.i18n.messages.MessagesFactory; +import org.apache.hadoop.gateway.services.security.AliasService; +import org.apache.hadoop.gateway.services.security.AliasServiceException; +import org.apache.hadoop.gateway.topology.discovery.GatewayService; +import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery; +import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryConfig; +import org.apache.http.HttpEntity; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.message.BasicHeader; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.util.*; + + +class AmbariServiceDiscovery implements ServiceDiscovery { + + static final String TYPE = "AMBARI"; + + static final String AMBARI_CLUSTERS_URI = "/api/v1/clusters"; + + static final String AMBARI_HOSTROLES_URI = + AMBARI_CLUSTERS_URI + "/%s/services?fields=components/host_components/HostRoles"; + + static final String AMBARI_SERVICECONFIGS_URI = + AMBARI_CLUSTERS_URI + "/%s/configurations/service_config_versions?is_current=true"; + + // Map of component names to service configuration types + private static Map<String, String> componentServiceConfigs = new HashMap<>(); + static { + componentServiceConfigs.put("NAMENODE", "hdfs-site"); + componentServiceConfigs.put("RESOURCEMANAGER", "yarn-site"); + componentServiceConfigs.put("OOZIE_SERVER", "oozie-site"); + componentServiceConfigs.put("HIVE_SERVER", "hive-site"); + componentServiceConfigs.put("WEBHCAT_SERVER", "webhcat-site"); + componentServiceConfigs.put("HBASE_MASTER", "hbase-site"); + } // TODO: Are there other service components, for which the endpoints can be discovered via Ambari? + + private static final String DEFAULT_USER_ALIAS = "ambari.discovery.user"; + private static final String DEFAULT_PWD_ALIAS = "ambari.discovery.password"; + + private static AmbariServiceURLCreator urlCreator = new AmbariServiceURLCreator(); + + private AmbariServiceDiscoveryMessages log = MessagesFactory.get(AmbariServiceDiscoveryMessages.class); + + @GatewayService + private AliasService aliasService; + + private CloseableHttpClient httpClient = null; + + private Map<String, Map<String, String>> serviceConfiguration = new HashMap<>(); + + + AmbariServiceDiscovery() { + httpClient = org.apache.http.impl.client.HttpClients.createDefault(); + } + + + @Override + public String getType() { + return TYPE; + } + + + @Override + public Map<String, Cluster> discover(ServiceDiscoveryConfig config) { + Map<String, Cluster> clusters = new HashMap<String, Cluster>(); + + String discoveryAddress = config.getAddress(); + + // Invoke Ambari REST API to discover the available clusters + String clustersDiscoveryURL = String.format("%s" + AMBARI_CLUSTERS_URI, discoveryAddress); + + JSONObject json = invokeREST(clustersDiscoveryURL, config.getUser(), config.getPasswordAlias()); + + // Parse the cluster names from the response, and perform the cluster discovery + JSONArray clusterItems = (JSONArray) json.get("items"); + for (Object clusterItem : clusterItems) { + String clusterName = (String) ((JSONObject)((JSONObject) clusterItem).get("Clusters")).get("cluster_name"); + try { + Cluster c = discover(config, clusterName); + clusters.put(clusterName, c); + } catch (Exception e) { + log.clusterDiscoveryError(clusterName, e); + } + } + + return clusters; + } + + + @Override + public Cluster discover(ServiceDiscoveryConfig config, String clusterName) { + AmbariCluster cluster = new AmbariCluster(clusterName); + + Map<String, String> serviceComponents = new HashMap<>(); + + String discoveryAddress = config.getAddress(); + String discoveryUser = config.getUser(); + String discoveryPwdAlias = config.getPasswordAlias(); + + Map<String, List<String>> componentHostNames = new HashMap<>(); + String hostRolesURL = String.format("%s" + AMBARI_HOSTROLES_URI, discoveryAddress, clusterName); + JSONObject hostRolesJSON = invokeREST(hostRolesURL, discoveryUser, discoveryPwdAlias); + if (hostRolesJSON != null) { + // Process the host roles JSON + JSONArray items = (JSONArray) hostRolesJSON.get("items"); + for (Object obj : items) { + JSONArray components = (JSONArray) ((JSONObject) obj).get("components"); + for (Object component : components) { + JSONArray hostComponents = (JSONArray) ((JSONObject) component).get("host_components"); + for (Object hostComponent : hostComponents) { + JSONObject hostRoles = (JSONObject) ((JSONObject) hostComponent).get("HostRoles"); + String serviceName = (String) hostRoles.get("service_name"); + String componentName = (String) hostRoles.get("component_name"); + + serviceComponents.put(componentName, serviceName); + +// String hostName = (String) hostRoles.get("host_name"); + String hostName = (String) hostRoles.get("public_host_name"); // Assuming public host name is most applicable + log.discoveredServiceHost(serviceName, hostName); + if (!componentHostNames.containsKey(componentName)) { + componentHostNames.put(componentName, new ArrayList<String>()); + } + componentHostNames.get(componentName).add(hostName); + } + } + } + } + + Map<String, Map<String, AmbariCluster.ServiceConfiguration>> serviceConfigurations = + new HashMap<String, Map<String, AmbariCluster.ServiceConfiguration>>(); + String serviceConfigsURL = String.format("%s" + AMBARI_SERVICECONFIGS_URI, discoveryAddress, clusterName); + JSONObject serviceConfigsJSON = invokeREST(serviceConfigsURL, discoveryUser, discoveryPwdAlias); + if (serviceConfigsJSON != null) { + // Process the service configurations + JSONArray serviceConfigs = (JSONArray) serviceConfigsJSON.get("items"); + for (Object serviceConfig : serviceConfigs) { + String serviceName = (String) ((JSONObject) serviceConfig).get("service_name"); + JSONArray configurations = (JSONArray) ((JSONObject) serviceConfig).get("configurations"); + for (Object configuration : configurations) { + String configType = (String) ((JSONObject) configuration).get("type"); + String configVersion = String.valueOf(((JSONObject) configuration).get("version")); + + Map<String, String> configProps = new HashMap<String, String>(); + JSONObject configProperties = (JSONObject) ((JSONObject) configuration).get("properties"); + for (String propertyName : configProperties.keySet()) { + configProps.put(propertyName, String.valueOf(((JSONObject) configProperties).get(propertyName))); + } + if (!serviceConfigurations.containsKey(serviceName)) { + serviceConfigurations.put(serviceName, new HashMap<String, AmbariCluster.ServiceConfiguration>()); + } + serviceConfigurations.get(serviceName).put(configType, new AmbariCluster.ServiceConfiguration(configType, configVersion, configProps)); + cluster.addServiceConfiguration(serviceName, configType, new AmbariCluster.ServiceConfiguration(configType, configVersion, configProps)); + } + } + } + + // Construct the AmbariCluster model + for (String componentName : serviceComponents.keySet()) { + String serviceName = serviceComponents.get(componentName); + List<String> hostNames = componentHostNames.get(componentName); + + Map<String, AmbariCluster.ServiceConfiguration> configs = serviceConfigurations.get(serviceName); + String configType = componentServiceConfigs.get(componentName); + if (configType != null) { + AmbariCluster.ServiceConfiguration svcConfig = configs.get(configType); + AmbariComponent c = new AmbariComponent(componentName, + svcConfig.getVersion(), + clusterName, + serviceName, + hostNames, + svcConfig.getProperties()); + cluster.addComponent(c); + } + } + + return cluster; + } + + + protected JSONObject invokeREST(String url, String username, String passwordAlias) { + JSONObject result = null; + + CloseableHttpResponse response = null; + try { + HttpGet request = new HttpGet(url); + + // If no configured username, then use default username alias + String password = null; + if (username == null) { + if (aliasService != null) { + try { + char[] defaultUser = aliasService.getPasswordFromAliasForGateway(DEFAULT_USER_ALIAS); + if (defaultUser != null) { + username = new String(defaultUser); + } + } catch (AliasServiceException e) { + log.aliasServiceUserError(DEFAULT_USER_ALIAS, e.getLocalizedMessage()); + } + } + + // If username is still null + if (username == null) { + log.aliasServiceUserNotFound(); + throw new ConfigurationException("No username is configured for Ambari service discovery."); + } + } + + if (aliasService != null) { + // If not password alias is configured, then try the default alias + if (passwordAlias == null) { + passwordAlias = DEFAULT_PWD_ALIAS; + } + try { + char[] pwd = aliasService.getPasswordFromAliasForGateway(passwordAlias); + if (pwd != null) { + password = new String(pwd); + } + + } catch (AliasServiceException e) { + log.aliasServicePasswordError(passwordAlias, e.getLocalizedMessage()); + } + } + + // If the password could not be determined + if (password == null) { + log.aliasServicePasswordNotFound(); + throw new ConfigurationException("No password is configured for Ambari service discovery."); + } + + // Add an auth header if credentials are available + String encodedCreds = + org.apache.commons.codec.binary.Base64.encodeBase64String((username + ":" + password).getBytes()); + request.addHeader(new BasicHeader("Authorization", "Basic " + encodedCreds)); + + response = httpClient.execute(request); + + if (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) { + HttpEntity entity = response.getEntity(); + if (entity != null) { + result = (JSONObject) JSONValue.parse((EntityUtils.toString(entity))); + log.debugJSON(result.toJSONString()); + } else { + log.noJSON(url); + } + } else { + log.unexpectedRestResponseStatusCode(url, response.getStatusLine().getStatusCode()); + } + + } catch (IOException e) { + log.restInvocationError(url, e); + } finally { + if(response != null) { + try { + response.close(); + } catch (IOException e) { + // Ignore + } + } + } + return result; + } + + +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java ---------------------------------------------------------------------- diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java new file mode 100644 index 0000000..caa16ed --- /dev/null +++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryMessages.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.discovery.ambari; + +import org.apache.hadoop.gateway.i18n.messages.Message; +import org.apache.hadoop.gateway.i18n.messages.MessageLevel; +import org.apache.hadoop.gateway.i18n.messages.Messages; +import org.apache.hadoop.gateway.i18n.messages.StackTrace; + +@Messages(logger="org.apache.gateway.topology.discovery.ambari") +public interface AmbariServiceDiscoveryMessages { + + @Message(level = MessageLevel.ERROR, + text = "Encountered an error during cluster {0} discovery: {1}") + void clusterDiscoveryError(final String clusterName, + @StackTrace(level = MessageLevel.ERROR) Exception e); + + + @Message(level = MessageLevel.DEBUG, + text = "REST invocation {0} failed: {1}") + void restInvocationError(final String url, + @StackTrace(level = MessageLevel.ERROR) Exception e); + + + @Message(level = MessageLevel.ERROR, + text = "Encountered an error attempting to determine the user for alias {0} : {1}") + void aliasServiceUserError(final String alias, final String error); + + + @Message(level = MessageLevel.ERROR, + text = "Encountered an error attempting to determine the password for alias {0} : {1}") + void aliasServicePasswordError(final String alias, final String error); + + + @Message(level = MessageLevel.ERROR, + text = "No user configured for Ambari service discovery.") + void aliasServiceUserNotFound(); + + + @Message(level = MessageLevel.ERROR, + text = "No password configured for Ambari service discovery.") + void aliasServicePasswordNotFound(); + + + @Message(level = MessageLevel.ERROR, + text = "Unexpected REST invocation response code for {0} : {1}") + void unexpectedRestResponseStatusCode(final String url, int responseStatusCode); + + + @Message(level = MessageLevel.ERROR, + text = "REST invocation {0} yielded a response without any JSON.") + void noJSON(final String url); + + + @Message(level = MessageLevel.DEBUG, + text = "REST invocation result: {0}") + void debugJSON(final String json); + + + @Message(level = MessageLevel.INFO, + text = "Discovered: Service: {0}, Host: {1}") + void discoveredServiceHost(final String serviceName, final String hostName); + + + + +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java ---------------------------------------------------------------------- diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java new file mode 100644 index 0000000..723a786 --- /dev/null +++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryType.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.discovery.ambari; + +import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery; +import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType; + +public class AmbariServiceDiscoveryType implements ServiceDiscoveryType { + + private static final String IMPL = AmbariServiceDiscovery.class.getCanonicalName(); + + @Override + public String getType() { + return AmbariServiceDiscovery.TYPE; + } + + @Override + public ServiceDiscovery newInstance() { + return new AmbariServiceDiscovery(); + } +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java ---------------------------------------------------------------------- diff --git a/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java new file mode 100644 index 0000000..0674642 --- /dev/null +++ b/b/gateway-discovery-ambari/src/main/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceURLCreator.java @@ -0,0 +1,184 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.gateway.topology.discovery.ambari; + + +import java.util.ArrayList; +import java.util.List; + +class AmbariServiceURLCreator { + + private static final String NAMENODE_SERVICE = "NAMENODE"; + private static final String JOBTRACKER_SERVICE = "JOBTRACKER"; + private static final String WEBHDFS_SERVICE = "WEBHDFS"; + private static final String WEBHCAT_SERVICE = "WEBHCAT"; + private static final String OOZIE_SERVICE = "OOZIE"; + private static final String WEBHBASE_SERVICE = "WEBHBASE"; + private static final String HIVE_SERVICE = "HIVE"; + private static final String RESOURCEMANAGER_SERVICE = "RESOURCEMANAGER"; + + + /** + * Derive the endpoint URL(s) for the specified service, based on the info from the specified Cluster. + * + * @param cluster The cluster discovery results + * @param serviceName The name of a Hadoop service + * + * @return One or more endpoint URLs for the specified service. + */ + public List<String> create(AmbariCluster cluster, String serviceName) { + List<String> result = null; + + if (NAMENODE_SERVICE.equals(serviceName)) { + result = createNameNodeURL(cluster); + } else if (JOBTRACKER_SERVICE.equals(serviceName)) { + result = createJobTrackerURL(cluster); + } else if (WEBHDFS_SERVICE.equals(serviceName)) { + result = createWebHDFSURL(cluster); + } else if (WEBHCAT_SERVICE.equals(serviceName)) { + result = createWebHCatURL(cluster); + } else if (OOZIE_SERVICE.equals(serviceName)) { + result = createOozieURL(cluster); + } else if (WEBHBASE_SERVICE.equals(serviceName)) { + result = createWebHBaseURL(cluster); + } else if (HIVE_SERVICE.equals(serviceName)) { + result = createHiveURL(cluster); + } else if (RESOURCEMANAGER_SERVICE.equals(serviceName)) { + result = createResourceManagerURL(cluster); + } + + return result; + } + + + private List<String> createNameNodeURL(AmbariCluster cluster) { + List<String> result = new ArrayList<>(); + + AmbariComponent comp = cluster.getComponent("NAMENODE"); + if (comp != null) { + result.add("hdfs://" + comp.getConfigProperty("dfs.namenode.rpc-address")); + } + + return result; + } + + + private List<String> createJobTrackerURL(AmbariCluster cluster) { + List<String> result = new ArrayList<>(); + + AmbariComponent comp = cluster.getComponent("RESOURCEMANAGER"); + if (comp != null) { + result.add("rpc://" + comp.getConfigProperty("yarn.resourcemanager.address")); + } + + return result; + } + + + private List<String> createWebHDFSURL(AmbariCluster cluster) { + List<String> result = new ArrayList<>(); + + AmbariCluster.ServiceConfiguration sc = cluster.getServiceConfiguration("HDFS", "hdfs-site"); + if (sc != null) { + String address = sc.getProperties().get("dfs.namenode.http-address"); + result.add("http://"; + address + "/webhdfs"); + } + + return result; + } + + + private List<String> createWebHCatURL(AmbariCluster cluster) { + List<String> result = new ArrayList<>(); + + AmbariComponent webhcat = cluster.getComponent("WEBHCAT_SERVER"); + if (webhcat != null) { + String port = webhcat.getConfigProperty("templeton.port"); + String host = webhcat.getHostNames().get(0); + + result.add("http://"; + host + ":" + port + "/templeton"); + } + return result; + } + + + private List<String> createOozieURL(AmbariCluster cluster) { + List<String> result = new ArrayList<>(); + + AmbariComponent comp = cluster.getComponent("OOZIE_SERVER"); + if (comp != null) { + result.add(comp.getConfigProperty("oozie.base.url")); + } + + return result; + } + + + private List<String> createWebHBaseURL(AmbariCluster cluster) { + List<String> result = new ArrayList<>(); + + AmbariComponent comp = cluster.getComponent("HBASE_MASTER"); + if (comp != null) { + for (String host : comp.getHostNames()) { + result.add("http://"; + host + ":60080"); + } + } + + return result; + } + + + private List<String> createHiveURL(AmbariCluster cluster) { + List<String> result = new ArrayList<>(); + + AmbariComponent hive = cluster.getComponent("HIVE_SERVER"); + if (hive != null) { + String path = hive.getConfigProperty("hive.server2.thrift.http.path"); + String port = hive.getConfigProperty("hive.server2.thrift.http.port"); + String transport = hive.getConfigProperty("hive.server2.transport.mode"); + String useSSL = hive.getConfigProperty("hive.server2.use.SSL"); + String host = hive.getHostNames().get(0); + + String scheme = null; // What is the scheme for the binary transport mode? + if ("http".equals(transport)) { + scheme = Boolean.valueOf(useSSL) ? "https" : "http"; + } + + result.add(scheme + "://" + host + ":" + port + "/" + path); + } + return result; + } + + + private List<String> createResourceManagerURL(AmbariCluster cluster) { + List<String> result = new ArrayList<>(); + + AmbariComponent resMan = cluster.getComponent("RESOURCEMANAGER"); + if (resMan != null) { + String webappAddress = resMan.getConfigProperty("yarn.resourcemanager.webapp.address"); + String httpPolicy = resMan.getConfigProperty("yarn.http.policy"); + String scheme = ("HTTPS_ONLY".equalsIgnoreCase(httpPolicy)) ? "https" : "http"; + + result.add(scheme + "://" + webappAddress + "/ws"); + } + + return result; + } + + +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType ---------------------------------------------------------------------- diff --git a/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType b/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType new file mode 100644 index 0000000..1da4fc9 --- /dev/null +++ b/b/gateway-discovery-ambari/src/main/resources/META-INF/services/org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryType @@ -0,0 +1,19 @@ +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## + +org.apache.hadoop.gateway.topology.discovery.ambari.AmbariServiceDiscoveryType \ No newline at end of file
