http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java ---------------------------------------------------------------------- diff --git a/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java new file mode 100644 index 0000000..1e5e7b2 --- /dev/null +++ b/gateway-discovery-ambari/src/test/java/org/apache/hadoop/gateway/topology/discovery/ambari/AmbariServiceDiscoveryTest.java @@ -0,0 +1,856 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.discovery.ambari; + +import net.minidev.json.JSONObject; +import net.minidev.json.JSONValue; +import org.apache.hadoop.gateway.topology.discovery.ServiceDiscovery; +import org.apache.hadoop.gateway.topology.discovery.ServiceDiscoveryConfig; +import org.easymock.EasyMock; +import org.junit.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.*; + + +/** + * Test the Ambari ServiceDiscovery implementation. + * + * N.B. These tests do NOT verify Ambari API responses. They DO validate the Ambari ServiceDiscovery implementation's + * treatment of the responses as they were observed at the time the tests are developed. + */ +public class AmbariServiceDiscoveryTest { + + @Test + public void testSingleClusterDiscovery() throws Exception { + final String discoveryAddress = "http://ambarihost:8080"; + final String clusterName = "testCluster"; + ServiceDiscovery sd = new TestAmbariServiceDiscovery(clusterName); + + ServiceDiscoveryConfig sdc = EasyMock.createNiceMock(ServiceDiscoveryConfig.class); + EasyMock.expect(sdc.getAddress()).andReturn(discoveryAddress).anyTimes(); + EasyMock.expect(sdc.getUser()).andReturn(null).anyTimes(); + EasyMock.replay(sdc); + + ServiceDiscovery.Cluster cluster = sd.discover(sdc, clusterName); + assertNotNull(cluster); + assertEquals(clusterName, cluster.getName()); + assertTrue(AmbariCluster.class.isAssignableFrom(cluster.getClass())); + assertEquals(6, ((AmbariCluster) cluster).getComponents().size()); + +// printServiceURLs(cluster); + } + + + @Test + public void testBulkClusterDiscovery() throws Exception { + final String discoveryAddress = "http://ambarihost:8080"; + final String clusterName = "anotherCluster"; + ServiceDiscovery sd = new TestAmbariServiceDiscovery(clusterName); + + ServiceDiscoveryConfig sdc = EasyMock.createNiceMock(ServiceDiscoveryConfig.class); + EasyMock.expect(sdc.getAddress()).andReturn(discoveryAddress).anyTimes(); + EasyMock.expect(sdc.getUser()).andReturn(null).anyTimes(); + EasyMock.replay(sdc); + + Map<String, ServiceDiscovery.Cluster> clusters = sd.discover(sdc); + assertNotNull(clusters); + assertEquals(1, clusters.size()); + ServiceDiscovery.Cluster cluster = clusters.get(clusterName); + assertNotNull(cluster); + assertEquals(clusterName, cluster.getName()); + assertTrue(AmbariCluster.class.isAssignableFrom(cluster.getClass())); + assertEquals(6, ((AmbariCluster) cluster).getComponents().size()); + +// printServiceURLs(cluster, "NAMENODE", "WEBHCAT", "OOZIE", "RESOURCEMANAGER"); + } + + + private static void printServiceURLs(ServiceDiscovery.Cluster cluster) { + final String[] services = new String[]{"NAMENODE", + "JOBTRACKER", + "WEBHDFS", + "WEBHCAT", + "OOZIE", + "WEBHBASE", + "HIVE", + "RESOURCEMANAGER"}; + printServiceURLs(cluster, services); + } + + + private static void printServiceURLs(ServiceDiscovery.Cluster cluster, String...services) { + for (String name : services) { + StringBuilder sb = new StringBuilder(); + List<String> urls = cluster.getServiceURLs(name); + if (urls != null && !urls.isEmpty()) { + for (String url : urls) { + sb.append(url); + sb.append(" "); + } + } + System.out.println(String.format("%18s: %s", name, sb.toString())); + } + } + + + /** + * ServiceDiscovery implementation derived from AmbariServiceDiscovery, so the invokeREST method can be overridden + * to eliminate the need to perform actual HTTP interactions with a real Ambari endpoint. + */ + private static final class TestAmbariServiceDiscovery extends AmbariServiceDiscovery { + + final static String CLUSTER_PLACEHOLDER = "CLUSTER_NAME"; + + private Map<String, JSONObject> cannedResponses = new HashMap<>(); + + TestAmbariServiceDiscovery(String clusterName) { + cannedResponses.put(AmbariServiceDiscovery.AMBARI_CLUSTERS_URI, + (JSONObject) JSONValue.parse(CLUSTERS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, + clusterName))); + + cannedResponses.put(String.format(AmbariServiceDiscovery.AMBARI_HOSTROLES_URI, clusterName), + (JSONObject) JSONValue.parse(HOSTROLES_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, + clusterName))); + + cannedResponses.put(String.format(AmbariServiceDiscovery.AMBARI_SERVICECONFIGS_URI, clusterName), + (JSONObject) JSONValue.parse(SERVICECONFIGS_JSON_TEMPLATE.replaceAll(CLUSTER_PLACEHOLDER, + clusterName))); + } + + @Override + protected JSONObject invokeREST(String url, String username, String passwordAlias) { + return cannedResponses.get(url.substring(url.indexOf("/api"))); + } + } + + + //////////////////////////////////////////////////////////////////////// + // JSON response templates, based on actual response content excerpts + //////////////////////////////////////////////////////////////////////// + + private static final String CLUSTERS_JSON_TEMPLATE = + "{\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters\",\n" + + " \"items\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"Clusters\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"version\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]" + + "}"; + + + private static final String HOSTROLES_JSON_TEMPLATE = + "{\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services?fields=components/host_components/HostRoles\",\n" + + " \"items\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/AMBARI_METRICS\",\n" + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"AMBARI_METRICS\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/AMBARI_METRICS/components/METRICS_COLLECTOR\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"METRICS_COLLECTOR\",\n" + + " \"service_name\" : \"AMBARI_METRICS\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/METRICS_COLLECTOR\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"METRICS_COLLECTOR\",\n" + + " \"host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"service_name\" : \"AMBARI_METRICS\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HBASE/components/HBASE_MASTER\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HBASE_MASTER\",\n" + + " \"service_name\" : \"HBASE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/HBASE_MASTER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HBASE_MASTER\",\n" + + " \"host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"service_name\" : \"HBASE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS\",\n" + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"HDFS\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS/components/NAMENODE\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"NAMENODE\",\n" + + " \"service_name\" : \"HDFS\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/NAMENODE\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"NAMENODE\",\n" + + " \"host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"service_name\" : \"HDFS\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HDFS/components/SECONDARY_NAMENODE\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"SECONDARY_NAMENODE\",\n" + + " \"service_name\" : \"HDFS\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/SECONDARY_NAMENODE\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"SECONDARY_NAMENODE\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"HDFS\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE\",\n" + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HCAT\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HCAT\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/HCAT\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HCAT\",\n" + + " \"host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HIVE_METASTORE\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HIVE_METASTORE\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/HIVE_METASTORE\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HIVE_METASTORE\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/HIVE_SERVER\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HIVE_SERVER\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/HIVE_SERVER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"HIVE_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/HIVE/components/WEBHCAT_SERVER\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"WEBHCAT_SERVER\",\n" + + " \"service_name\" : \"HIVE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/WEBHCAT_SERVER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"WEBHCAT_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/OOZIE\",\n" + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"OOZIE\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/OOZIE/components/OOZIE_SERVER\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"OOZIE_SERVER\",\n" + + " \"service_name\" : \"OOZIE\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/OOZIE_SERVER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"OOZIE_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"OOZIE\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN\",\n" + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"YARN\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/APP_TIMELINE_SERVER\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"APP_TIMELINE_SERVER\",\n" + + " \"service_name\" : \"YARN\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/APP_TIMELINE_SERVER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"APP_TIMELINE_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"YARN\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/NODEMANAGER\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"NODEMANAGER\",\n" + + " \"service_name\" : \"YARN\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/NODEMANAGER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"NODEMANAGER\",\n" + + " \"host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"service_name\" : \"YARN\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/YARN/components/RESOURCEMANAGER\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"RESOURCEMANAGER\",\n" + + " \"service_name\" : \"YARN\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/RESOURCEMANAGER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"RESOURCEMANAGER\",\n" + + " \"ha_state\" : \"ACTIVE\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"YARN\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/ZOOKEEPER\",\n" + + " \"ServiceInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"service_name\" : \"ZOOKEEPER\"\n" + + " },\n" + + " \"components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/services/ZOOKEEPER/components/ZOOKEEPER_SERVER\",\n" + + " \"ServiceComponentInfo\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" + + " \"service_name\" : \"ZOOKEEPER\"\n" + + " },\n" + + " \"host_components\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6401.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" + + " \"host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6401.ambari.apache.org\",\n" + + " \"service_name\" : \"ZOOKEEPER\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6402.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" + + " \"host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6402.ambari.apache.org\",\n" + + " \"service_name\" : \"ZOOKEEPER\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/hosts/c6403.ambari.apache.org/host_components/ZOOKEEPER_SERVER\",\n" + + " \"HostRoles\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"component_name\" : \"ZOOKEEPER_SERVER\",\n" + + " \"host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"public_host_name\" : \"c6403.ambari.apache.org\",\n" + + " \"service_name\" : \"ZOOKEEPER\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " }\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}\n"; + + + private static final String SERVICECONFIGS_JSON_TEMPLATE = + "{\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?is_current=true\",\n" + + " \"items\" : [\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HBASE&service_config_version=1\",\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hbase-site\",\n" + + " \"tag\" : \"version1503410563715\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hbase.master.info.bindAddress\" : \"0.0.0.0\",\n" + + " \"hbase.master.info.port\" : \"16010\",\n" + + " \"hbase.master.port\" : \"16000\",\n" + + " \"hbase.regionserver.info.port\" : \"16030\",\n" + + " \"hbase.regionserver.port\" : \"16020\",\n" + + " \"hbase.zookeeper.property.clientPort\" : \"2181\",\n" + + " \"hbase.zookeeper.quorum\" : \"c6403.ambari.apache.org,c6402.ambari.apache.org,c6401.ambari.apache.org\",\n" + + " \"hbase.zookeeper.useMulti\" : \"true\",\n" + + " \"zookeeper.znode.parent\" : \"/hbase-unsecure\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " ],\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 1,\n" + + " \"service_config_version_note\" : \"Initial configurations for HBase\",\n" + + " \"service_name\" : \"HBASE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HDFS&service_config_version=2\",\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hdfs-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"dfs.cluster.administrators\" : \" hdfs\",\n" + + " \"dfs.datanode.address\" : \"0.0.0.0:50010\",\n" + + " \"dfs.datanode.http.address\" : \"0.0.0.0:50075\",\n" + + " \"dfs.datanode.https.address\" : \"0.0.0.0:50475\",\n" + + " \"dfs.datanode.ipc.address\" : \"0.0.0.0:8010\",\n" + + " \"dfs.http.policy\" : \"HTTP_ONLY\",\n" + + " \"dfs.https.port\" : \"50470\",\n" + + " \"dfs.journalnode.http-address\" : \"0.0.0.0:8480\",\n" + + " \"dfs.journalnode.https-address\" : \"0.0.0.0:8481\",\n" + + " \"dfs.namenode.http-address\" : \"c6401.ambari.apache.org:50070\",\n" + + " \"dfs.namenode.https-address\" : \"c6401.ambari.apache.org:50470\",\n" + + " \"dfs.namenode.rpc-address\" : \"c6401.ambari.apache.org:8020\",\n" + + " \"dfs.namenode.secondary.http-address\" : \"c6402.ambari.apache.org:50090\",\n" + + " \"dfs.webhdfs.enabled\" : \"true\"\n" + + " },\n" + + " \"properties_attributes\" : {\n" + + " \"final\" : {\n" + + " \"dfs.webhdfs.enabled\" : \"true\",\n" + + " \"dfs.namenode.http-address\" : \"true\",\n" + + " \"dfs.support.append\" : \"true\",\n" + + " \"dfs.namenode.name.dir\" : \"true\",\n" + + " \"dfs.datanode.failed.volumes.tolerated\" : \"true\",\n" + + " \"dfs.datanode.data.dir\" : \"true\"\n" + + " }\n" + + " }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"core-site\",\n" + + " \"tag\" : \"version1502131215159\",\n" + + " \"version\" : 2,\n" + + " \"properties\" : {\n" + + " \"hadoop.http.authentication.simple.anonymous.allowed\" : \"true\",\n" + + " \"net.topology.script.file.name\" : \"/etc/hadoop/conf/topology_script.py\"\n" + + " },\n" + + " \"properties_attributes\" : {\n" + + " \"final\" : {\n" + + " \"fs.defaultFS\" : \"true\"\n" + + " }\n" + + " }\n" + + " }\n" + + " ],\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 2,\n" + + " \"service_config_version_note\" : \"knox trusted proxy support\",\n" + + " \"service_name\" : \"HDFS\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=HIVE&service_config_version=3\",\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hive-env\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hive_security_authorization\" : \"None\",\n" + + " \"webhcat_user\" : \"hcat\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hiveserver2-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hive.metastore.metrics.enabled\" : \"true\",\n" + + " \"hive.security.authorization.enabled\" : \"false\",\n" + + " \"hive.service.metrics.hadoop2.component\" : \"hiveserver2\",\n" + + " \"hive.service.metrics.reporter\" : \"HADOOP2\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hive-interactive-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hive.server2.enable.doAs\" : \"false\",\n" + + " \"hive.server2.tez.default.queues\" : \"default\",\n" + + " \"hive.server2.tez.initialize.default.sessions\" : \"true\",\n" + + " \"hive.server2.tez.sessions.custom.queue.allowed\" : \"ignore\",\n" + + " \"hive.server2.tez.sessions.per.default.queue\" : \"1\",\n" + + " \"hive.server2.tez.sessions.restricted.configs\" : \"hive.execution.mode,hive.execution.engine\",\n" + + " \"hive.server2.thrift.http.port\" : \"10501\",\n" + + " \"hive.server2.thrift.port\" : \"10500\",\n" + + " \"hive.server2.webui.port\" : \"10502\",\n" + + " \"hive.server2.webui.use.ssl\" : \"false\",\n" + + " \"hive.server2.zookeeper.namespace\" : \"hiveserver2-hive2\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"tez-interactive-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"tez.am.am-rm.heartbeat.interval-ms.max\" : \"10000\",\n" + + " \"tez.am.client.heartbeat.poll.interval.millis\" : \"6000\",\n" + + " \"tez.am.client.heartbeat.timeout.secs\" : \"90\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"hive-site\",\n" + + " \"tag\" : \"version1502130841736\",\n" + + " \"version\" : 2,\n" + + " \"properties\" : {\n" + + " \"hive.metastore.sasl.enabled\" : \"false\",\n" + + " \"hive.metastore.server.max.threads\" : \"100000\",\n" + + " \"hive.metastore.uris\" : \"thrift://c6402.ambari.apache.org:9083\",\n" + + " \"hive.server2.allow.user.substitution\" : \"true\",\n" + + " \"hive.server2.authentication\" : \"NONE\",\n" + + " \"hive.server2.authentication.spnego.keytab\" : \"HTTP/_h...@example.com\",\n" + + " \"hive.server2.authentication.spnego.principal\" : \"/etc/security/keytabs/spnego.service.keytab\",\n" + + " \"hive.server2.enable.doAs\" : \"true\",\n" + + " \"hive.server2.support.dynamic.service.discovery\" : \"true\",\n" + + " \"hive.server2.thrift.http.path\" : \"cliservice\",\n" + + " \"hive.server2.thrift.http.port\" : \"10001\",\n" + + " \"hive.server2.thrift.max.worker.threads\" : \"500\",\n" + + " \"hive.server2.thrift.port\" : \"10000\",\n" + + " \"hive.server2.thrift.sasl.qop\" : \"auth\",\n" + + " \"hive.server2.transport.mode\" : \"http\",\n" + + " \"hive.server2.use.SSL\" : \"false\",\n" + + " \"hive.server2.zookeeper.namespace\" : \"hiveserver2\"\n" + + " },\n" + + " \"properties_attributes\" : {\n" + + " \"hidden\" : {\n" + + " \"javax.jdo.option.ConnectionPassword\" : \"HIVE_CLIENT,WEBHCAT_SERVER,HCAT,CONFIG_DOWNLOAD\"\n" + + " }\n" + + " }\n" + + " },\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"webhcat-site\",\n" + + " \"tag\" : \"version1502131111746\",\n" + + " \"version\" : 2,\n" + + " \"properties\" : {\n" + + " \"templeton.port\" : \"50111\",\n" + + " \"templeton.zookeeper.hosts\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\",\n" + + " \"webhcat.proxyuser.knox.groups\" : \"users\",\n" + + " \"webhcat.proxyuser.knox.hosts\" : \"*\",\n" + + " \"webhcat.proxyuser.root.groups\" : \"*\",\n" + + " \"webhcat.proxyuser.root.hosts\" : \"c6401.ambari.apache.org\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " }\n" + + " ],\n" + + " \"createtime\" : 1502131110745,\n" + + " \"group_id\" : -1,\n" + + " \"group_name\" : \"Default\",\n" + + " \"hosts\" : [ ],\n" + + " \"is_cluster_compatible\" : true,\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 3,\n" + + " \"service_config_version_note\" : \"knox trusted proxy support\",\n" + + " \"service_name\" : \"HIVE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=OOZIE&service_config_version=3\",\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"oozie-site\",\n" + + " \"tag\" : \"version1502131137103\",\n" + + " \"version\" : 3,\n" + + " \"properties\" : {\n" + + " \"oozie.base.url\" : \"http://c6402.ambari.apache.org:11000/oozie\",\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " }\n" + + " ],\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 3,\n" + + " \"service_name\" : \"OOZIE\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=TEZ&service_config_version=1\",\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"tez-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"tez.use.cluster.hadoop-libs\" : \"false\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " }\n" + + " ],\n" + + " \"createtime\" : 1502122253525,\n" + + " \"group_id\" : -1,\n" + + " \"group_name\" : \"Default\",\n" + + " \"hosts\" : [ ],\n" + + " \"is_cluster_compatible\" : true,\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 1,\n" + + " \"service_config_version_note\" : \"Initial configurations for Tez\",\n" + + " \"service_name\" : \"TEZ\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " },\n" + + " {\n" + + " \"href\" : \"http://c6401.ambari.apache.org:8080/api/v1/clusters/"+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"/configurations/service_config_versions?service_name=YARN&service_config_version=1\",\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"configurations\" : [\n" + + " {\n" + + " \"Config\" : {\n" + + " \"cluster_name\" : \""+TestAmbariServiceDiscovery.CLUSTER_PLACEHOLDER+"\",\n" + + " \"stack_id\" : \"HDP-2.6\"\n" + + " },\n" + + " \"type\" : \"yarn-site\",\n" + + " \"tag\" : \"version1\",\n" + + " \"version\" : 1,\n" + + " \"properties\" : {\n" + + " \"hadoop.registry.rm.enabled\" : \"true\",\n" + + " \"hadoop.registry.zk.quorum\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\",\n" + + " \"yarn.acl.enable\" : \"false\",\n" + + " \"yarn.http.policy\" : \"HTTP_ONLY\",\n" + + " \"yarn.nodemanager.address\" : \"0.0.0.0:45454\",\n" + + " \"yarn.nodemanager.bind-host\" : \"0.0.0.0\",\n" + + " \"yarn.resourcemanager.address\" : \"c6402.ambari.apache.org:8050\",\n" + + " \"yarn.resourcemanager.admin.address\" : \"c6402.ambari.apache.org:8141\",\n" + + " \"yarn.resourcemanager.ha.enabled\" : \"false\",\n" + + " \"yarn.resourcemanager.hostname\" : \"c6402.ambari.apache.org\",\n" + + " \"yarn.resourcemanager.resource-tracker.address\" : \"c6402.ambari.apache.org:8025\",\n" + + " \"yarn.resourcemanager.scheduler.address\" : \"c6402.ambari.apache.org:8030\",\n" + + " \"yarn.resourcemanager.webapp.address\" : \"c6402.ambari.apache.org:8088\",\n" + + " \"yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled\" : \"false\",\n" + + " \"yarn.resourcemanager.webapp.https.address\" : \"c6402.ambari.apache.org:8090\",\n" + + " \"yarn.resourcemanager.zk-address\" : \"c6403.ambari.apache.org:2181,c6401.ambari.apache.org:2181,c6402.ambari.apache.org:2181\"\n" + + " },\n" + + " \"properties_attributes\" : { }\n" + + " }\n" + + " ],\n" + + " \"is_current\" : true,\n" + + " \"service_config_version\" : 1,\n" + + " \"service_name\" : \"YARN\",\n" + + " \"stack_id\" : \"HDP-2.6\",\n" + + " \"user\" : \"admin\"\n" + + " }\n" + + " ]\n" + + "}"; + +}
http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/home/conf/descriptors/README ---------------------------------------------------------------------- diff --git a/gateway-release/home/conf/descriptors/README b/gateway-release/home/conf/descriptors/README new file mode 100644 index 0000000..a2e5226 --- /dev/null +++ b/gateway-release/home/conf/descriptors/README @@ -0,0 +1 @@ +THIS DIRECTORY IS WHERE SIMPLE TOPOLOGY DESCRIPTORS CAN BE PLACED \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/home/conf/shared-providers/README ---------------------------------------------------------------------- diff --git a/gateway-release/home/conf/shared-providers/README b/gateway-release/home/conf/shared-providers/README new file mode 100644 index 0000000..44d12a3 --- /dev/null +++ b/gateway-release/home/conf/shared-providers/README @@ -0,0 +1 @@ +THIS DIRECTORY IS WHERE SHARED PROVIDER CONFIGURATIONS CAN BE PLACED \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-release/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml index 83824cd..cbff307 100644 --- a/gateway-release/pom.xml +++ b/gateway-release/pom.xml @@ -334,6 +334,10 @@ <groupId>${gateway-group}</groupId> <artifactId>gateway-shell-samples</artifactId> </dependency> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-discovery-ambari</artifactId> + </dependency> <dependency> <groupId>junit</groupId> http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-server/pom.xml b/gateway-server/pom.xml index 437d22d..5ebf793 100644 --- a/gateway-server/pom.xml +++ b/gateway-server/pom.xml @@ -248,6 +248,11 @@ <artifactId>javax-websocket-client-impl</artifactId> </dependency> + <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-yaml</artifactId> + <version>2.3.0</version> + </dependency> <!-- ********** ********** ********** ********** ********** ********** --> <!-- ********** Test Dependencies ********** --> http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java index 1f94584..6f73c1e 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/GatewayMessages.java @@ -511,6 +511,11 @@ public interface GatewayMessages { " Could not find topology \"{0}\" mapped to port \"{1}\" configured in gateway-config.xml. " + "This invalid topology mapping will be ignored by the gateway. " + "Gateway restart will be required if in the future \"{0}\" topology is added.") - void topologyPortMappingCannotFindTopology(final String topology, - final int port); + void topologyPortMappingCannotFindTopology(final String topology, final int port); + + + @Message( level = MessageLevel.ERROR, text = "An error occurred while processing {0} : {1}" ) + void simpleDescriptorHandlingError(final String simpleDesc, + @StackTrace( level = MessageLevel.DEBUG ) Exception e ); + } http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java index cefada1..02ac154 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/DefaultGatewayServices.java @@ -105,14 +105,13 @@ public class DefaultGatewayServices implements GatewayServices { services.put( SERVER_INFO_SERVICE, sis ); DefaultTopologyService tops = new DefaultTopologyService(); + tops.setAliasService(alias); tops.init( config, options ); services.put( TOPOLOGY_SERVICE, tops ); DefaultServiceDefinitionRegistry sdr = new DefaultServiceDefinitionRegistry(); sdr.init( config, options ); services.put( SERVICE_DEFINITION_REGISTRY, sdr ); - tops.init( config, options ); - services.put( TOPOLOGY_SERVICE, tops ); DefaultMetricsService metricsService = new DefaultMetricsService(); metricsService.init( config, options ); http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java index 3321f3d..a493bc4 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java @@ -38,6 +38,7 @@ import org.apache.hadoop.gateway.config.GatewayConfig; import org.apache.hadoop.gateway.i18n.messages.MessagesFactory; import org.apache.hadoop.gateway.service.definition.ServiceDefinition; import org.apache.hadoop.gateway.services.ServiceLifecycleException; +import org.apache.hadoop.gateway.services.security.AliasService; import org.apache.hadoop.gateway.services.topology.TopologyService; import org.apache.hadoop.gateway.topology.Topology; import org.apache.hadoop.gateway.topology.TopologyEvent; @@ -45,6 +46,7 @@ import org.apache.hadoop.gateway.topology.TopologyListener; import org.apache.hadoop.gateway.topology.TopologyMonitor; import org.apache.hadoop.gateway.topology.TopologyProvider; import org.apache.hadoop.gateway.topology.builder.TopologyBuilder; +import org.apache.hadoop.gateway.topology.simple.SimpleDescriptorHandler; import org.apache.hadoop.gateway.topology.validation.TopologyValidator; import org.apache.hadoop.gateway.topology.xml.AmbariFormatXmlTopologyRules; import org.apache.hadoop.gateway.topology.xml.KnoxFormatXmlTopologyRules; @@ -74,20 +76,27 @@ import static org.apache.commons.digester3.binder.DigesterLoader.newLoader; public class DefaultTopologyService extends FileAlterationListenerAdaptor implements TopologyService, TopologyMonitor, TopologyProvider, FileFilter, FileAlterationListener { + private static Auditor auditor = AuditServiceFactory.getAuditService().getAuditor( AuditConstants.DEFAULT_AUDITOR_NAME, AuditConstants.KNOX_SERVICE_NAME, AuditConstants.KNOX_COMPONENT_NAME); + private static final List<String> SUPPORTED_TOPOLOGY_FILE_EXTENSIONS = new ArrayList<String>(); static { SUPPORTED_TOPOLOGY_FILE_EXTENSIONS.add("xml"); SUPPORTED_TOPOLOGY_FILE_EXTENSIONS.add("conf"); } + private static GatewayMessages log = MessagesFactory.get(GatewayMessages.class); private static DigesterLoader digesterLoader = newLoader(new KnoxFormatXmlTopologyRules(), new AmbariFormatXmlTopologyRules()); - private FileAlterationMonitor monitor; - private File directory; + private List<FileAlterationMonitor> monitors = new ArrayList<>(); + private File topologiesDirectory; + private File descriptorsDirectory; + private Set<TopologyListener> listeners; private volatile Map<File, Topology> topologies; + private AliasService aliasService; + private Topology loadTopology(File file) throws IOException, SAXException, URISyntaxException, InterruptedException { final long TIMEOUT = 250; //ms @@ -202,29 +211,40 @@ public class DefaultTopologyService } private File calculateAbsoluteTopologiesDir(GatewayConfig config) { - - File topoDir = new File(config.getGatewayTopologyDir()); + String normalizedTopologyDir = FilenameUtils.normalize(config.getGatewayTopologyDir()); + File topoDir = new File(normalizedTopologyDir); topoDir = topoDir.getAbsoluteFile(); return topoDir; } - private void initListener(FileAlterationMonitor monitor, File directory) { - this.directory = directory; - this.monitor = monitor; + private File calculateAbsoluteConfigDir(GatewayConfig config) { + File configDir = null; + String path = FilenameUtils.normalize(config.getGatewayConfDir()); + if (path != null) { + configDir = new File(config.getGatewayConfDir()); + } else { + configDir = (new File(config.getGatewayTopologyDir())).getParentFile(); + } + configDir = configDir.getAbsoluteFile(); - FileAlterationObserver observer = new FileAlterationObserver(this.directory, this); - observer.addListener(this); - monitor.addObserver(observer); + return configDir; + } - this.listeners = new HashSet<>(); - this.topologies = new HashMap<>(); //loadTopologies( this.directory ); + private void initListener(FileAlterationMonitor monitor, + File directory, + FileFilter filter, + FileAlterationListener listener) { + monitors.add(monitor); + FileAlterationObserver observer = new FileAlterationObserver(directory, filter); + observer.addListener(listener); + monitor.addObserver(observer); } - private void initListener(File directory) throws IOException, SAXException { + private void initListener(File directory, FileFilter filter, FileAlterationListener listener) throws IOException, SAXException { // Increasing the monitoring interval to 5 seconds as profiling has shown // this is rather expensive in terms of generated garbage objects. - initListener(new FileAlterationMonitor(5000L), directory); + initListener(new FileAlterationMonitor(5000L), directory, filter, listener); } private Map<File, Topology> loadTopologies(File directory) { @@ -261,10 +281,14 @@ public class DefaultTopologyService return map; } + public void setAliasService(AliasService as) { + this.aliasService = as; + } + public void deployTopology(Topology t){ try { - File temp = new File(directory.getAbsolutePath() + "/" + t.getName() + ".xml.temp"); + File temp = new File(topologiesDirectory.getAbsolutePath() + "/" + t.getName() + ".xml.temp"); Package topologyPkg = Topology.class.getPackage(); String pkgName = topologyPkg.getName(); String bindingFile = pkgName.replace(".", "/") + "/topology_binding-xml.xml"; @@ -277,7 +301,7 @@ public class DefaultTopologyService mr.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); mr.marshal(t, temp); - File topology = new File(directory.getAbsolutePath() + "/" + t.getName() + ".xml"); + File topology = new File(topologiesDirectory.getAbsolutePath() + "/" + t.getName() + ".xml"); if(!temp.renameTo(topology)) { FileUtils.forceDelete(temp); throw new IOException("Could not rename temp file"); @@ -317,7 +341,7 @@ public class DefaultTopologyService try { synchronized (this) { Map<File, Topology> oldTopologies = topologies; - Map<File, Topology> newTopologies = loadTopologies(directory); + Map<File, Topology> newTopologies = loadTopologies(topologiesDirectory); List<TopologyEvent> events = createChangeEvents(oldTopologies, newTopologies); topologies = newTopologies; notifyChangeListeners(events); @@ -329,7 +353,7 @@ public class DefaultTopologyService } public void deleteTopology(Topology t) { - File topoDir = directory; + File topoDir = topologiesDirectory; if(topoDir.isDirectory() && topoDir.canRead()) { File[] results = topoDir.listFiles(); @@ -357,8 +381,8 @@ public class DefaultTopologyService public Map<String, List<String>> getServiceTestURLs(Topology t, GatewayConfig config) { File tFile = null; Map<String, List<String>> urls = new HashMap<>(); - if(directory.isDirectory() && directory.canRead()) { - for(File f : directory.listFiles()){ + if(topologiesDirectory.isDirectory() && topologiesDirectory.canRead()) { + for(File f : topologiesDirectory.listFiles()){ if(FilenameUtils.removeExtension(f.getName()).equals(t.getName())){ tFile = f; } @@ -387,12 +411,16 @@ public class DefaultTopologyService @Override public void startMonitor() throws Exception { - monitor.start(); + for (FileAlterationMonitor monitor : monitors) { + monitor.start(); + } } @Override public void stopMonitor() throws Exception { - monitor.stop(); + for (FileAlterationMonitor monitor : monitors) { + monitor.stop(); + } } @Override @@ -414,6 +442,16 @@ public class DefaultTopologyService @Override public void onFileDelete(java.io.File file) { + // For full topology descriptors, we need to make sure to delete any corresponding simple descriptors to prevent + // unintended subsequent generation of the topology descriptor + for (String ext : DescriptorsMonitor.SUPPORTED_EXTENSIONS) { + File simpleDesc = + new File(descriptorsDirectory, FilenameUtils.getBaseName(file.getName()) + "." + ext); + if (simpleDesc.exists()) { + simpleDesc.delete(); + } + } + onFileChange(file); } @@ -436,12 +474,200 @@ public class DefaultTopologyService public void init(GatewayConfig config, Map<String, String> options) throws ServiceLifecycleException { try { - initListener(calculateAbsoluteTopologiesDir(config)); - } catch (IOException io) { + listeners = new HashSet<>(); + topologies = new HashMap<>(); + + topologiesDirectory = calculateAbsoluteTopologiesDir(config); + + File configDirectory = calculateAbsoluteConfigDir(config); + descriptorsDirectory = new File(configDirectory, "descriptors"); + File sharedProvidersDirectory = new File(configDirectory, "shared-providers"); + + // Add support for conf/topologies + initListener(topologiesDirectory, this, this); + + // Add support for conf/descriptors + DescriptorsMonitor dm = new DescriptorsMonitor(topologiesDirectory, aliasService); + initListener(descriptorsDirectory, + dm, + dm); + + // Add support for conf/shared-providers + SharedProviderConfigMonitor spm = new SharedProviderConfigMonitor(dm, descriptorsDirectory); + initListener(sharedProvidersDirectory, spm, spm); + + } catch (IOException | SAXException io) { throw new ServiceLifecycleException(io.getMessage()); - } catch (SAXException sax) { - throw new ServiceLifecycleException(sax.getMessage()); + } + } + + + /** + * Change handler for simple descriptors + */ + public static class DescriptorsMonitor extends FileAlterationListenerAdaptor + implements FileFilter { + + static final List<String> SUPPORTED_EXTENSIONS = new ArrayList<String>(); + static { + SUPPORTED_EXTENSIONS.add("json"); + SUPPORTED_EXTENSIONS.add("yml"); + } + + private File topologiesDir; + + private AliasService aliasService; + + private Map<String, List<String>> providerConfigReferences = new HashMap<>(); + + + public DescriptorsMonitor(File topologiesDir, AliasService aliasService) { + this.topologiesDir = topologiesDir; + this.aliasService = aliasService; + } + + List<String> getReferencingDescriptors(String providerConfigPath) { + List<String> result = providerConfigReferences.get(providerConfigPath); + if (result == null) { + result = Collections.emptyList(); + } + return result; + } + + @Override + public void onFileCreate(File file) { + onFileChange(file); + } + + @Override + public void onFileDelete(File file) { + // For simple descriptors, we need to make sure to delete any corresponding full topology descriptors to trigger undeployment + for (String ext : DefaultTopologyService.SUPPORTED_TOPOLOGY_FILE_EXTENSIONS) { + File topologyFile = + new File(topologiesDir, FilenameUtils.getBaseName(file.getName()) + "." + ext); + if (topologyFile.exists()) { + topologyFile.delete(); + } + } + + String normalizedFilePath = FilenameUtils.normalize(file.getAbsolutePath()); + String reference = null; + for (Map.Entry<String, List<String>> entry : providerConfigReferences.entrySet()) { + if (entry.getValue().contains(normalizedFilePath)) { + reference = entry.getKey(); + break; + } + } + if (reference != null) { + providerConfigReferences.get(reference).remove(normalizedFilePath); + } + } + + @Override + public void onFileChange(File file) { + try { + // When a simple descriptor has been created or modified, generate the new topology descriptor + Map<String, File> result = SimpleDescriptorHandler.handle(file, topologiesDir, aliasService); + + // Add the provider config reference relationship for handling updates to the provider config + String providerConfig = FilenameUtils.normalize(result.get("reference").getAbsolutePath()); + if (!providerConfigReferences.containsKey(providerConfig)) { + providerConfigReferences.put(providerConfig, new ArrayList<String>()); + } + List<String> refs = providerConfigReferences.get(providerConfig); + String descriptorName = FilenameUtils.normalize(file.getAbsolutePath()); + if (!refs.contains(descriptorName)) { + // Need to check if descriptor had previously referenced another provider config, so it can be removed + for (List<String> descs : providerConfigReferences.values()) { + if (descs.contains(descriptorName)) { + descs.remove(descriptorName); + } + } + + // Add the current reference relationship + refs.add(descriptorName); + } + } catch (Exception e) { + log.simpleDescriptorHandlingError(file.getName(), e); + } + } + + @Override + public boolean accept(File file) { + boolean accept = false; + if (!file.isDirectory() && file.canRead()) { + String extension = FilenameUtils.getExtension(file.getName()); + if (SUPPORTED_EXTENSIONS.contains(extension)) { + accept = true; + } + } + return accept; + } + } + + /** + * Change handler for shared provider configurations + */ + public static class SharedProviderConfigMonitor extends FileAlterationListenerAdaptor + implements FileFilter { + + static final List<String> SUPPORTED_EXTENSIONS = new ArrayList<>(); + static { + SUPPORTED_EXTENSIONS.add("xml"); } + private DescriptorsMonitor descriptorsMonitor; + private File descriptorsDir; + + + SharedProviderConfigMonitor(DescriptorsMonitor descMonitor, File descriptorsDir) { + this.descriptorsMonitor = descMonitor; + this.descriptorsDir = descriptorsDir; + } + + @Override + public void onFileCreate(File file) { + onFileChange(file); + } + + @Override + public void onFileDelete(File file) { + onFileChange(file); + } + + @Override + public void onFileChange(File file) { + // For shared provider configuration, we need to update any simple descriptors that reference it + for (File descriptor : getReferencingDescriptors(file)) { + descriptor.setLastModified(System.currentTimeMillis()); + } + } + + private List<File> getReferencingDescriptors(File sharedProviderConfig) { + List<File> references = new ArrayList<>(); + + for (File descriptor : descriptorsDir.listFiles()) { + if (DescriptorsMonitor.SUPPORTED_EXTENSIONS.contains(FilenameUtils.getExtension(descriptor.getName()))) { + for (String reference : descriptorsMonitor.getReferencingDescriptors(FilenameUtils.normalize(sharedProviderConfig.getAbsolutePath()))) { + references.add(new File(reference)); + } + } + } + + return references; + } + + @Override + public boolean accept(File file) { + boolean accept = false; + if (!file.isDirectory() && file.canRead()) { + String extension = FilenameUtils.getExtension(file.getName()); + if (SUPPORTED_EXTENSIONS.contains(extension)) { + accept = true; + } + } + return accept; + } } + } http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java index b5e80d2..a30cf13 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/builder/BeanPropertyTopologyBuilder.java @@ -77,7 +77,7 @@ public class BeanPropertyTopologyBuilder implements TopologyBuilder { Topology topology = new Topology(); topology.setName(name); - for (Provider provider : providers) { + for (Provider provider : providers) { topology.addProvider(provider); } http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java new file mode 100644 index 0000000..6534b5e --- /dev/null +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/DefaultServiceDiscoveryConfig.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.discovery; + +public class DefaultServiceDiscoveryConfig implements ServiceDiscoveryConfig { + private String address = null; + private String user = null; + private String pwdAlias = null; + + public DefaultServiceDiscoveryConfig(String address) { + this.address = address; + } + + public void setUser(String username) { + this.user = username; + } + + public void setPasswordAlias(String alias) { + this.pwdAlias = alias; + } + + public String getAddress() { + return address; + } + + public String getUser() { + return user; + } + + public String getPasswordAlias() { + return pwdAlias; + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java new file mode 100644 index 0000000..70d5f61 --- /dev/null +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/discovery/ServiceDiscoveryFactory.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.discovery; + +import org.apache.hadoop.gateway.services.Service; + +import java.lang.reflect.Field; +import java.util.ServiceLoader; + +/** + * Creates instances of ServiceDiscovery implementations. + * + * This factory uses the ServiceLoader mechanism to load ServiceDiscovery implementations as extensions. + * + */ +public abstract class ServiceDiscoveryFactory { + + private static final Service[] NO_GATEWAY_SERVICS = new Service[]{}; + + + public static ServiceDiscovery get(String type) { + return get(type, NO_GATEWAY_SERVICS); + } + + + public static ServiceDiscovery get(String type, Service...gatewayServices) { + ServiceDiscovery sd = null; + + // Look up the available ServiceDiscovery types + ServiceLoader<ServiceDiscoveryType> loader = ServiceLoader.load(ServiceDiscoveryType.class); + for (ServiceDiscoveryType sdt : loader) { + if (sdt.getType().equalsIgnoreCase(type)) { + try { + ServiceDiscovery instance = sdt.newInstance(); + // Make sure the type reported by the instance matches the type declared by the factory + // (is this necessary?) + if (instance.getType().equalsIgnoreCase(type)) { + sd = instance; + + // Inject any gateway services that were specified, and which are referenced in the impl + if (gatewayServices != null && gatewayServices.length > 0) { + for (Field field : sd.getClass().getDeclaredFields()) { + if (field.getDeclaredAnnotation(GatewayService.class) != null) { + for (Service s : gatewayServices) { + if (s != null) { + if (field.getType().isAssignableFrom(s.getClass())) { + field.setAccessible(true); + field.set(sd, s); + } + } + } + } + } + } + break; + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + return sd; + } + + +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java new file mode 100644 index 0000000..aa28469 --- /dev/null +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptor.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.simple; + +import java.util.List; + +public interface SimpleDescriptor { + + String getName(); + + String getDiscoveryType(); + + String getDiscoveryAddress(); + + String getDiscoveryUser(); + + String getDiscoveryPasswordAlias(); + + String getClusterName(); + + String getProviderConfig(); + + List<Service> getServices(); + + + interface Service { + String getName(); + + List<String> getURLs(); + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/c2ca4432/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java new file mode 100644 index 0000000..3df6d2f --- /dev/null +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/topology/simple/SimpleDescriptorFactory.java @@ -0,0 +1,71 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.gateway.topology.simple; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.commons.io.FilenameUtils; + +import java.io.File; +import java.io.IOException; + + +public class SimpleDescriptorFactory { + + /** + * Create a SimpleDescriptor from the specified file. + * + * @param path The path to the file. + * @return A SimpleDescriptor based on the contents of the file. + * + * @throws IOException + */ + public static SimpleDescriptor parse(String path) throws IOException { + SimpleDescriptor sd; + + if (path.endsWith(".json")) { + sd = parseJSON(path); + } else if (path.endsWith(".yml")) { + sd = parseYAML(path); + } else { + throw new IllegalArgumentException("Unsupported simple descriptor format: " + path.substring(path.lastIndexOf('.'))); + } + + return sd; + } + + + static SimpleDescriptor parseJSON(String path) throws IOException { + final ObjectMapper mapper = new ObjectMapper(); + SimpleDescriptorImpl sd = mapper.readValue(new File(path), SimpleDescriptorImpl.class); + if (sd != null) { + sd.setName(FilenameUtils.getBaseName(path)); + } + return sd; + } + + + static SimpleDescriptor parseYAML(String path) throws IOException { + final ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + SimpleDescriptorImpl sd = mapper.readValue(new File(path), SimpleDescriptorImpl.class); + if (sd != null) { + sd.setName(FilenameUtils.getBaseName(path)); + } + return sd; + } + +}