Repository: knox Updated Branches: refs/heads/master 06a61b602 -> fae40583d
KNOX-549: Test service connections through Knox with Knox CLI Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/fae40583 Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/fae40583 Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/fae40583 Branch: refs/heads/master Commit: fae40583d026e94a77f3bf6168bb53a917c24b38 Parents: 06a61b6 Author: Kevin Minder <kevin.min...@hortonworks.com> Authored: Tue Jul 21 17:42:42 2015 -0400 Committer: Kevin Minder <kevin.min...@hortonworks.com> Committed: Tue Jul 21 17:42:42 2015 -0400 ---------------------------------------------------------------------- CHANGES | 1 + gateway-release/pom.xml | 4 + .../topology/impl/DefaultTopologyService.java | 21 + .../org/apache/hadoop/gateway/util/KnoxCLI.java | 190 ++++++++- .../gateway/util/ServiceDefinitionsLoader.java | 52 ++- .../service/definition/ServiceDefinition.java | 18 + .../resources/services/falcon/0.6.0/service.xml | 7 + .../resources/services/hbase/0.98.0/service.xml | 6 + .../resources/services/oozie/4.0.0/service.xml | 5 + .../resources/services/storm/0.9.3/service.xml | 6 + .../services/webhcat/0.13.0/service.xml | 6 + .../services/webhdfs/2.4.0/service.xml | 3 + .../services/yarn-rm/2.5.0/service.xml | 5 + gateway-service-test/pom.xml | 63 +++ .../service/test/ServiceTestResource.java | 421 +++++++++++++++++++ .../gateway/service/test/ServiceTestURL.java | 89 ++++ .../test/ServiceTestWrapperMarshaller.java | 72 ++++ .../ServiceTestDeploymentContributor.java | 102 +++++ ....gateway.deploy.ServiceDeploymentContributor | 19 + .../hadoop/gateway/service/test/jaxb.properties | 16 + .../services/topology/TopologyService.java | 7 +- .../hadoop/gateway/GatewayBasicFuncTest.java | 246 ++++++++++- .../hadoop/gateway/GatewayFuncTestDriver.java | 30 ++ pom.xml | 6 + 24 files changed, 1368 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index 40f2757..ed7bdc7 100644 --- a/CHANGES +++ b/CHANGES @@ -5,6 +5,7 @@ Release Notes - Apache Knox - Version 0.7.0 * [KNOX-560] - Test LDAP Authentication+Authorization from KnoxCLI * [KNOX-547] - KnoxCLI adds new validate-topology and list-topologies commands. * [KNOX-548] - KnoxCLI adds a new system-user-auth-test command to test a topology's system username and password + * [KNOX-549] - New Service-Test API can be added to topology. Accessible via Http call or KnoxCLI ** Improvement * [KNOX-553] - Added topology validation from KnoxCLI to TopologyService deployment. http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-release/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-release/pom.xml b/gateway-release/pom.xml index 33472cf..b95865f 100644 --- a/gateway-release/pom.xml +++ b/gateway-release/pom.xml @@ -152,6 +152,10 @@ </dependency> <dependency> <groupId>${gateway-group}</groupId> + <artifactId>gateway-service-test</artifactId> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> <artifactId>gateway-service-as</artifactId> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java index d298ba0..ecfbcd9 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/services/topology/impl/DefaultTopologyService.java @@ -36,6 +36,7 @@ import org.apache.hadoop.gateway.audit.api.ResourceType; import org.apache.hadoop.gateway.audit.log4j.audit.AuditConstants; import org.apache.hadoop.gateway.config.GatewayConfig; import org.apache.hadoop.gateway.i18n.messages.MessagesFactory; +import org.apache.hadoop.gateway.service.definition.ServiceDefinition; import org.apache.hadoop.gateway.services.ServiceLifecycleException; import org.apache.hadoop.gateway.services.topology.TopologyService; import org.apache.hadoop.gateway.topology.Topology; @@ -47,6 +48,7 @@ import org.apache.hadoop.gateway.topology.builder.TopologyBuilder; import org.apache.hadoop.gateway.topology.validation.TopologyValidator; import org.apache.hadoop.gateway.topology.xml.AmbariFormatXmlTopologyRules; import org.apache.hadoop.gateway.topology.xml.KnoxFormatXmlTopologyRules; +import org.apache.hadoop.gateway.util.ServiceDefinitionsLoader; import org.eclipse.persistence.jaxb.JAXBContextProperties; import org.xml.sax.SAXException; @@ -350,6 +352,25 @@ public class DefaultTopologyService } } + public Map<String, List<String>> getServiceTestURLs(Topology t, GatewayConfig config) { + File tFile = null; + Map<String, List<String>> urls = new HashMap<>(); + for(File f : directory.listFiles()){ + if(FilenameUtils.removeExtension(f.getName()).equals(t.getName())){ + tFile = f; + } + } + Set<ServiceDefinition> defs; + if(tFile != null) { + defs = ServiceDefinitionsLoader.getServiceDefinitions(new File(config.getGatewayServicesDir())); + + for(ServiceDefinition def : defs) { + urls.put(def.getRole(), def.getTestURLs()); + } + } + return urls; + } + public Collection<Topology> getTopologies() { Map<File, Topology> map = topologies; return Collections.unmodifiableCollection(map.values()); http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java index 9022921..497121e 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/util/KnoxCLI.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.gateway.util; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; @@ -38,6 +39,13 @@ import org.apache.hadoop.gateway.topology.Topology; import org.apache.hadoop.gateway.topology.validation.TopologyValidator; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; import org.apache.log4j.PropertyConfigurator; import org.apache.shiro.SecurityUtils; import org.apache.shiro.authc.AuthenticationException; @@ -45,13 +53,18 @@ import org.apache.shiro.authc.UsernamePasswordToken; import org.apache.shiro.config.IniSecurityManagerFactory; import org.apache.shiro.subject.Subject; import org.apache.shiro.util.Factory; +import org.eclipse.persistence.oxm.MediaType; import org.jboss.shrinkwrap.api.exporter.ExplodedExporter; import org.jboss.shrinkwrap.api.spec.WebArchive; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLException; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.PrintStream; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -81,7 +94,8 @@ public class KnoxCLI extends Configured implements Tool { " [" + ListTopologiesCommand.USAGE + "]\n" + " [" + ValidateTopologyCommand.USAGE + "]\n" + " [" + LDAPAuthCommand.USAGE + "]\n" + - " [" + LDAPSysBindCommand.USAGE + "]\n"; + " [" + LDAPSysBindCommand.USAGE + "]\n" + + " [" + ServiceTestCommand.USAGE + "]\n"; /** allows stdout to be captured if necessary */ public PrintStream out = System.out; @@ -95,6 +109,7 @@ public class KnoxCLI extends Configured implements Tool { private String path = null; private String generate = "false"; private String hostname = null; + private String port = null; private boolean force = false; private boolean debug = false; private String user = null; @@ -163,6 +178,8 @@ public class KnoxCLI extends Configured implements Tool { * % knoxcli validate-topology [--cluster clustername] | [--path <path/to/file>] * % knoxcli user-auth-test [--cluster clustername] [--u username] [--p password] * % knoxcli system-user-auth-test [--cluster clustername] [--d] + * % knoxcli service-test [--u user] [--p password] [--cluster clustername] [--hostname name] [--port port] + * * </pre> * @param args * @return @@ -251,6 +268,13 @@ public class KnoxCLI extends Configured implements Tool { return -1; } this.cluster = args[++i]; + } else if (args[i].equals("service-test")) { + if( i + 1 >= args[i].length()) { + printKnoxShellUsage(); + return -1; + } else { + command = new ServiceTestCommand(); + } } else if (args[i].equals("--generate")) { if ( command != null && command instanceof MasterCreateCommand ) { this.master = UUID.randomUUID().toString(); @@ -269,6 +293,12 @@ public class KnoxCLI extends Configured implements Tool { return -1; } this.hostname = args[++i]; + } else if (args[i].equals("--port")) { + if( i+1 >= args.length || args[i+1].startsWith( "-" ) ) { + printKnoxShellUsage(); + return -1; + } + this.port = args[++i]; } else if (args[i].equals("--master")) { // For testing only if( i+1 >= args.length || args[i+1].startsWith( "-" ) ) { @@ -351,6 +381,9 @@ public class KnoxCLI extends Configured implements Tool { out.println(LDAPSysBindCommand.USAGE + "\n\n" + LDAPSysBindCommand.DESC); out.println(); out.println( div ); + out.println(ServiceTestCommand.USAGE + "\n\n" + ServiceTestCommand.DESC); + out.println(); + out.println( div ); } } @@ -1294,6 +1327,161 @@ public class KnoxCLI extends Configured implements Tool { } + public class ServiceTestCommand extends Command { + public static final String USAGE = "service-test [--u username] [--p password] [--cluster clustername] [--hostname name] " + + "[--port port]"; + public static final String DESC = "This command requires a running instance of Knox to be present on the same " + + "machine. It will execute a test to make sure all services are accessible through the gateway URLs. Errors are " + + "reported and suggestions to resolve any problems are returned. JSON formatted."; + + private boolean ssl = true; + private int attempts = 0; + + @Override + public String getUsage() { return USAGE + ":\n\n" + DESC; }; + + @Override + public void execute() { + attempts++; + SSLContext ctx = null; + CloseableHttpClient client; + String http = "https://"; + String https = "http://"; + GatewayConfig conf = getGatewayConfig(); + String gatewayPort; + String host; + + + if(cluster == null) { + printKnoxShellUsage(); + out.println("A --cluster argument is required."); + return; + } + + if(hostname != null) { + host = hostname; + } else { + try { + host = InetAddress.getLocalHost().getHostAddress(); + } catch (UnknownHostException e) { + out.println(e.getMessage()); + out.println("Defaulting address to localhost. Use --hostname option to specify a different hostname"); + host = "localhost"; + } + } + + if (port != null) { + gatewayPort = port; + } else if (conf.getGatewayPort() > -1) { + gatewayPort = Integer.toString(conf.getGatewayPort()); + } else { + out.println("Could not get port. Please supply it using the --port option"); + return; + } + + + String path = "/" + conf.getGatewayPath(); + String topology = "/" + cluster; + String httpServiceTestURL = http + host + ":" + gatewayPort + path + topology + "/service-test"; + String httpsServiceTestURL = https + host + ":" + gatewayPort + path + topology + "/service-test"; + + String authString = ""; +// Create Authorization String + if( user != null && pass != null) { + authString = "Basic " + Base64.encodeBase64String((user + ":" + pass).getBytes()); + } else { + out.println("Username and/or password not supplied. Expect HTTP 401 Unauthorized responses."); + } + +// Attempt to build SSL context for HTTP client. + try { + ctx = SSLContexts.custom().loadTrustMaterial(null, new TrustSelfSignedStrategy()).build(); + } catch (Exception e) { + out.println(e.getMessage()); + } + +// Initialize the HTTP client + if(ctx == null) { + client = HttpClients.createDefault(); + } else { + client = HttpClients.custom().setSslcontext(ctx).build(); + } + + HttpGet request = new HttpGet(httpsServiceTestURL); + request.setHeader("Authorization", authString); + request.setHeader("Accept", MediaType.APPLICATION_JSON.getMediaType()); + try { + CloseableHttpResponse response = client.execute(request); + +// Fallback to http in case Http in case https doesn't work. + if(response.getStatusLine().getStatusCode() != 200) { + request = new HttpGet(httpServiceTestURL); + response = client.execute(request); + } + + response.close(); + request.releaseConnection(); + + switch (response.getStatusLine().getStatusCode()) { + + case 200: + response.getEntity().writeTo(out); + break; + case 404: + out.println("Could not find service-test resource"); + out.println("Make sure you have configured the SERVICE-TEST service in your topology."); + break; + case 500: + out.println("HTTP 500 Server error"); + break; + + default: + out.println("Unexpected HTTP response code."); + out.println(response.getStatusLine().toString()); + response.getEntity().writeTo(out); + break; + + + } + + + } catch (ClientProtocolException e) { + out.println(e.getMessage()); + if (debug) { + e.printStackTrace(out); + } + } catch (SSLException e) { + out.println(e.getMessage()); + + if(attempts < 2) { + if(ssl) { + ssl = false; + out.println("Attempting request without SSL."); + } else { + ssl = true; + out.println("Attempting request with SSL "); + } + execute(); + } else { + out.println("Unable to successfully make request. Try using the API with cURL."); + } + } catch (IOException e) { + out.println(e.getMessage()); + if(debug) { + e.printStackTrace(out); + } + } finally { + try { + client.close(); + } catch (IOException e) { + out.println(e.getMessage()); + } + } + + } + + } + private GatewayConfig getGatewayConfig() { GatewayConfig result; Configuration conf = getConf(); http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-server/src/main/java/org/apache/hadoop/gateway/util/ServiceDefinitionsLoader.java ---------------------------------------------------------------------- diff --git a/gateway-server/src/main/java/org/apache/hadoop/gateway/util/ServiceDefinitionsLoader.java b/gateway-server/src/main/java/org/apache/hadoop/gateway/util/ServiceDefinitionsLoader.java index 2ff07fd..0f41b2b 100644 --- a/gateway-server/src/main/java/org/apache/hadoop/gateway/util/ServiceDefinitionsLoader.java +++ b/gateway-server/src/main/java/org/apache/hadoop/gateway/util/ServiceDefinitionsLoader.java @@ -55,17 +55,8 @@ public class ServiceDefinitionsLoader { try { JAXBContext context = JAXBContext.newInstance(ServiceDefinition.class); Unmarshaller unmarshaller = context.createUnmarshaller(); - Collection<File> files = FileUtils.listFiles(servicesDir, new IOFileFilter() { - @Override - public boolean accept(File file) { - return file.getName().contains(SERVICE_FILE_NAME); - } - @Override - public boolean accept(File dir, String name) { - return name.contains(SERVICE_FILE_NAME); - } - }, TrueFileFilter.INSTANCE); - for ( File file : files ) { + + for ( File file : getFileList(servicesDir) ) { try { FileInputStream inputStream = new FileInputStream(file); ServiceDefinition definition = (ServiceDefinition) unmarshaller.unmarshal(inputStream); @@ -84,6 +75,45 @@ public class ServiceDefinitionsLoader { return contributors; } + public static Set<ServiceDefinition> getServiceDefinitions(File servicesDir) { + Set<ServiceDefinition> definitions = new HashSet<>(); + try { + JAXBContext context = JAXBContext.newInstance(ServiceDefinition.class); + Unmarshaller unmarshaller = context.createUnmarshaller(); + + for (File f : getFileList(servicesDir)){ + ServiceDefinition definition = (ServiceDefinition) unmarshaller.unmarshal(f); + definitions.add( definition ); + } + + } catch (JAXBException e) { + log.failedToLoadServiceDefinition(SERVICE_FILE_NAME, e); + } + + return definitions; + } + + private static Collection<File> getFileList(File servicesDir) { + Collection<File> files; + if ( servicesDir.exists() && servicesDir.isDirectory() ) { + files = FileUtils.listFiles(servicesDir, new IOFileFilter() { + @Override + public boolean accept(File file) { + return file.getName().contains(SERVICE_FILE_NAME); + } + + @Override + public boolean accept(File dir, String name) { + return name.contains(SERVICE_FILE_NAME); + } + }, TrueFileFilter.INSTANCE); + } else { + return files = new HashSet<File>(); + } + + return files; + } + private static UrlRewriteRulesDescriptor loadRewriteRules(File servicesDir) { File rewriteFile = new File(servicesDir, REWRITE_FILE); if ( rewriteFile.exists() ) { http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-definitions/src/main/java/org/apache/hadoop/gateway/service/definition/ServiceDefinition.java ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/java/org/apache/hadoop/gateway/service/definition/ServiceDefinition.java b/gateway-service-definitions/src/main/java/org/apache/hadoop/gateway/service/definition/ServiceDefinition.java index cbb9309..e85e6f9 100644 --- a/gateway-service-definitions/src/main/java/org/apache/hadoop/gateway/service/definition/ServiceDefinition.java +++ b/gateway-service-definitions/src/main/java/org/apache/hadoop/gateway/service/definition/ServiceDefinition.java @@ -21,6 +21,7 @@ import javax.xml.bind.annotation.XmlAttribute; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlElementWrapper; import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; import java.util.List; @XmlRootElement(name = "service") @@ -38,6 +39,8 @@ public class ServiceDefinition { private CustomDispatch dispatch; + private List<String> testURLs; + @XmlAttribute public String getName() { return name; @@ -90,6 +93,21 @@ public class ServiceDefinition { return dispatch; } + @XmlElement(name = "testURL") + @XmlElementWrapper(name = "testURLs") + public List<String> getTestURLs() { + + if(testURLs != null){ + return testURLs; + } else { + return new ArrayList<String>(); + } + } + + public void setTestURLs(List<String> testURLs) { + this.testURLs = testURLs; + } + public void setDispatch(CustomDispatch dispatch) { this.dispatch = dispatch; } http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-definitions/src/main/resources/services/falcon/0.6.0/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/falcon/0.6.0/service.xml b/gateway-service-definitions/src/main/resources/services/falcon/0.6.0/service.xml index 716f165..f4d4cf5 100644 --- a/gateway-service-definitions/src/main/resources/services/falcon/0.6.0/service.xml +++ b/gateway-service-definitions/src/main/resources/services/falcon/0.6.0/service.xml @@ -18,4 +18,11 @@ <routes> <route path="/falcon/api/**"/> </routes> + <testURLs> + <testURL>/falcon/api/admin/stack</testURL> + <testURL>/falcon/api/admin/version</testURL> + <testURL>/falcon/api/metadata/lineage/serialize</testURL> + <testURL>/falcon/api/metadata/lineage/vertices/all</testURL> + <testURL>/falcon/api/metadata/lineage/edges/all</testURL> + </testURLs> </service> http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-definitions/src/main/resources/services/hbase/0.98.0/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/hbase/0.98.0/service.xml b/gateway-service-definitions/src/main/resources/services/hbase/0.98.0/service.xml index 37d49bb..181b536 100644 --- a/gateway-service-definitions/src/main/resources/services/hbase/0.98.0/service.xml +++ b/gateway-service-definitions/src/main/resources/services/hbase/0.98.0/service.xml @@ -30,4 +30,10 @@ </route> </routes> <dispatch classname="org.apache.hadoop.gateway.hbase.HBaseDispatch"/> + <testURLs> + <testURL>/hbase/version</testURL> + <testURL>/hbase/version/cluster</testURL> + <testURL>/hbase/status/cluster</testURL> + <testURL>/hbase</testURL> + </testURLs> </service> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-definitions/src/main/resources/services/oozie/4.0.0/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/oozie/4.0.0/service.xml b/gateway-service-definitions/src/main/resources/services/oozie/4.0.0/service.xml index 7bc7eb3..455746c 100644 --- a/gateway-service-definitions/src/main/resources/services/oozie/4.0.0/service.xml +++ b/gateway-service-definitions/src/main/resources/services/oozie/4.0.0/service.xml @@ -27,4 +27,9 @@ <rewrite apply="OOZIE/oozie/configuration" to="request.body"/> </route> </routes> + <testURLs> + <testURL>/oozie/v1/admin/build-version</testURL> + <testURL>/oozie/v1/admin/status</testURL> + <testURL>/oozie/versions</testURL> + </testURLs> </service> http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-definitions/src/main/resources/services/storm/0.9.3/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/storm/0.9.3/service.xml b/gateway-service-definitions/src/main/resources/services/storm/0.9.3/service.xml index ffe20a4..c5f388e 100644 --- a/gateway-service-definitions/src/main/resources/services/storm/0.9.3/service.xml +++ b/gateway-service-definitions/src/main/resources/services/storm/0.9.3/service.xml @@ -25,4 +25,10 @@ </route> </routes> <dispatch classname="org.apache.hadoop.gateway.storm.StormDispatch"/> + <testURLs> + <testURL>/storm/api/v1/cluster/configuration</testURL> + <testURL>/storm/api/v1/cluster/summary</testURL> + <testURL>/storm/api/v1/supervisor/summary</testURL> + <testURL>/storm/api/v1/topology/summary</testURL> + </testURLs> </service> http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-definitions/src/main/resources/services/webhcat/0.13.0/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/webhcat/0.13.0/service.xml b/gateway-service-definitions/src/main/resources/services/webhcat/0.13.0/service.xml index c550dc1..d8c92c6 100644 --- a/gateway-service-definitions/src/main/resources/services/webhcat/0.13.0/service.xml +++ b/gateway-service-definitions/src/main/resources/services/webhcat/0.13.0/service.xml @@ -20,4 +20,10 @@ <route path="/templeton/v1/?**"/> <route path="/templeton/v1/**?**"/> </routes> + <testURLs> + <testURL>/templeton/v1/status</testURL> + <testURL>/templeton/v1/version</testURL> + <testURL>/templeton/v1/version/hive</testURL> + <testURL>/templeton/v1/version/hadoop</testURL> + </testURLs> </service> http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml index f958b42..9d39a32 100644 --- a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml +++ b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml @@ -37,4 +37,7 @@ </route> </routes> <dispatch classname="org.apache.hadoop.gateway.hdfs.dispatch.HdfsHttpClientDispatch" ha-classname="org.apache.hadoop.gateway.hdfs.dispatch.WebHdfsHaDispatch"/> + <testURLs> + <testURL>/webhdfs/v1/?op=LISTSTATUS</testURL> + </testURLs> </service> http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml index 73fb771..6b4c6a5 100644 --- a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml +++ b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml @@ -51,4 +51,9 @@ <rewrite apply="RESOURCEMANAGER/resourcemanager/proxy/taskattempt/outbound" to="response.body"/> </route> </routes> + <testURLs> + <testURL>/resourcemanager/v1/cluster/info</testURL> + <testURL>/resourcemanager/v1/cluster/metrics</testURL> + <testURL>/resourcemanager/v1/cluster/apps</testURL> + </testURLs> </service> http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-test/pom.xml ---------------------------------------------------------------------- diff --git a/gateway-service-test/pom.xml b/gateway-service-test/pom.xml new file mode 100644 index 0000000..357d23c --- /dev/null +++ b/gateway-service-test/pom.xml @@ -0,0 +1,63 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0" + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <parent> + <artifactId>gateway</artifactId> + <groupId>org.apache.knox</groupId> + <version>0.7.0-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + <artifactId>gateway-service-test</artifactId> + <version>0.7.0-SNAPSHOT</version> + <name>gateway-service-test</name> + <dependencies> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-spi</artifactId> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-provider-rewrite</artifactId> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> + <artifactId>gateway-provider-jersey</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>eclipselink</artifactId> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.knox</groupId> + <artifactId>gateway-test-utils</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <scope>test</scope> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestResource.java ---------------------------------------------------------------------- diff --git a/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestResource.java b/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestResource.java new file mode 100644 index 0000000..0d26301 --- /dev/null +++ b/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestResource.java @@ -0,0 +1,421 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.gateway.service.test; + +import org.apache.hadoop.gateway.config.GatewayConfig; +import org.apache.hadoop.gateway.services.GatewayServices; +import org.apache.hadoop.gateway.services.topology.TopologyService; +import org.apache.hadoop.gateway.topology.Service; +import org.apache.hadoop.gateway.topology.Topology; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.conn.ssl.SSLContexts; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.glassfish.jersey.internal.util.Base64; + +import javax.net.ssl.SSLContext; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.GET; +import javax.ws.rs.QueryParam; +import javax.ws.rs.PathParam; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.core.Context; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlElementWrapper; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static javax.ws.rs.core.MediaType.APPLICATION_JSON; +import static javax.ws.rs.core.MediaType.APPLICATION_XML; +import static javax.ws.rs.core.Response.Status.NOT_FOUND; +import static javax.ws.rs.core.Response.ok; +import static javax.ws.rs.core.Response.status; + +@Path( "/service-test" ) +public class ServiceTestResource { + @Context + private HttpServletRequest request; + + + @GET + @Produces({APPLICATION_XML, APPLICATION_JSON}) + public ServiceTestWrapper serviceTest(@QueryParam("username") String username, + @QueryParam("password") String password) { + List<ServiceTest> tests = new ArrayList<>(); + List<String> messages = new ArrayList<>(); + String authString; + GatewayConfig config = (GatewayConfig) request.getServletContext().getAttribute(GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE); + SSLContext ctx = null; + CloseableHttpClient client; + String id = getTopologyName(); + + Topology topology = getTopology(id); + +// Create Authorization String + if( username != null && password != null) { + authString = "Basic " + Base64.encodeAsString((username + ":" + password).getBytes()); + } else if (request.getHeader("Authorization") != null) { + authString = request.getHeader("Authorization"); + } else { + authString = null; + } + +// Attempt to build SSL context for HTTP client. + try { + ctx = SSLContexts.custom().loadTrustMaterial(null, new TrustSelfSignedStrategy()).build(); + } catch (Exception e) { + messages.add(e.getMessage()); + } + +// Initialize the HTTP client + if(ctx == null) { + client = HttpClients.createDefault(); + } else { + client = HttpClients.custom().setSslcontext(ctx).build(); + } + + if(topology != null) { + for (Service s : topology.getServices()) { + List<String> urls = getServiceTestURLs(config, s.getRole(), topology); + +// Make sure we handle a case where no URLs are found. + if(urls.size() <= 0) { + ServiceTest test = new ServiceTest(s); + test.setMessage("This service did not contain any test URLs"); + } + + for (String url : urls) { + HttpGet req = new HttpGet(); + ServiceTest test = new ServiceTest(s, url); + + if(authString != null) { + req.setHeader("Authorization", authString); + } else { + messages.add("No credentials provided. Expect HTTP 401 responses."); + } + + try { + req.setURI(new URIBuilder(url).build()); + CloseableHttpResponse res = client.execute(req); + String contentLength = "Content-Length:" + res.getEntity().getContentLength(); + String contentType = (res.getEntity().getContentType() != null) ? res.getEntity().getContentType().toString() : "No-contenttype"; + test.setResponseContent( contentLength + "," + contentType ); + test.setHttpCode(res.getStatusLine().getStatusCode()); + res.close(); + + } catch (IOException e) { + messages.add("Exception: " + e.getMessage()); + test.setMessage(e.getMessage()); + } catch (URISyntaxException e) { + test.setMessage(e.getMessage()); + } catch (Exception e) { + messages.add(e.getMessage()); + test.setMessage(e.getMessage()); + } finally { + req.releaseConnection(); + tests.add(test); + } + } + } + } else { + messages.add("Topology " + id + " not found"); + } + + try { + client.close(); + } catch (IOException e) { + e.printStackTrace(); + } + + ServiceTestWrapper stw = new ServiceTestWrapper(); + stw.setTests(tests); + stw.setMessages(messages); + + return stw; + } + + private String getTopologyName() { + String ctxPath = request.getContextPath(); + GatewayConfig config = (GatewayConfig) request.getServletContext().getAttribute(GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE); + String path = config.getGatewayPath(); + + String topologyName = ctxPath.replace(path, "").replace("/", ""); + return topologyName; + } + + public Topology getTopology(@PathParam("id") String id) { + GatewayServices services = (GatewayServices) request.getServletContext() + .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE); + GatewayConfig config = (GatewayConfig) request.getServletContext().getAttribute(GatewayConfig.GATEWAY_CONFIG_ATTRIBUTE); + + TopologyService ts = services.getService(GatewayServices.TOPOLOGY_SERVICE); + + for (Topology t : ts.getTopologies()) { + if(t.getName().equals(id)) { + try { + t.setUri(new URI( buildURI(t, config, request) )); + } catch (URISyntaxException se) { + t.setUri(null); + } + return t; + } + } + return null; + } + + private List<String> getServiceTestURLs(GatewayConfig conf, String role, Topology topology) { + GatewayServices services = (GatewayServices) request.getServletContext() + .getAttribute(GatewayServices.GATEWAY_SERVICES_ATTRIBUTE); + List<String> fullURLs = new ArrayList<>(); + if(services != null) { + TopologyService ts = services.getService(GatewayServices.TOPOLOGY_SERVICE); + Map<String, List<String>> urls = ts.getServiceTestURLs(topology, conf); + List<String> urlPaths = urls.get(role); + + if(urlPaths != null) { + String base = buildURI(topology, conf, request); + for (String u : urlPaths) { + + fullURLs.add(base + u); + } + } + } + return fullURLs; + } + + private String buildXForwardBaseURL(HttpServletRequest req){ + final String X_Forwarded = "X-Forwarded-"; + final String X_Forwarded_Context = X_Forwarded + "Context"; + final String X_Forwarded_Proto = X_Forwarded + "Proto"; + final String X_Forwarded_Host = X_Forwarded + "Host"; + final String X_Forwarded_Port = X_Forwarded + "Port"; + final String X_Forwarded_Server = X_Forwarded + "Server"; + + String baseURL = ""; + +// Get Protocol + if(req.getHeader(X_Forwarded_Proto) != null){ + baseURL += req.getHeader(X_Forwarded_Proto) + "://"; + } else { + baseURL += req.getProtocol() + "://"; + } + +// Handle Server/Host and Port Here + if (req.getHeader(X_Forwarded_Host) != null && req.getHeader(X_Forwarded_Port) != null){ +// Double check to see if host has port + if(req.getHeader(X_Forwarded_Host).contains(req.getHeader(X_Forwarded_Port))){ + baseURL += req.getHeader(X_Forwarded_Host); + } else { +// If there's no port, add the host and port together; + baseURL += req.getHeader(X_Forwarded_Host) + ":" + req.getHeader(X_Forwarded_Port); + } + } else if(req.getHeader(X_Forwarded_Server) != null && req.getHeader(X_Forwarded_Port) != null){ +// Tack on the server and port if they're available. Try host if server not available + baseURL += req.getHeader(X_Forwarded_Server) + ":" + req.getHeader(X_Forwarded_Port); + } else if(req.getHeader(X_Forwarded_Port) != null) { +// if we at least have a port, we can use it. + baseURL += req.getServerName() + ":" + req.getHeader(X_Forwarded_Port); + } else { +// Resort to request members + baseURL += req.getServerName() + ":" + req.getLocalPort(); + } + +// Handle Server context + if( req.getHeader(X_Forwarded_Context) != null ) { + baseURL += req.getHeader( X_Forwarded_Context ); + } else { + baseURL += req.getContextPath(); + } + + return baseURL; + } + + String buildURI(Topology topology, GatewayConfig config, HttpServletRequest req){ + String uri = buildXForwardBaseURL(req); + +// Strip extra context + uri = uri.replace(req.getContextPath(), ""); + +// Add the gateway path + String gatewayPath; + if(config.getGatewayPath() != null){ + gatewayPath = config.getGatewayPath(); + }else{ + gatewayPath = "gateway"; + } + uri += "/" + gatewayPath; + + uri += "/" + topology.getName(); + return uri; + } + + @XmlAccessorType(XmlAccessType.NONE) + public static class ServiceTest { + + @XmlElement + private String serviceName; + @XmlElement + private String requestURL; + @XmlElement + private String responseContent; + @XmlElement + private int httpCode = -1; + + @XmlElement + String message; + + public ServiceTest() { } + + public ServiceTest(Service s) { + this.serviceName = s.getRole(); + } + + public ServiceTest(Service s, String requestURL) { + this.serviceName = s.getRole(); + this.requestURL = requestURL; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String n) { + serviceName = n; + } + + public String getRequestURL() { + return requestURL; + } + + public void setRequestURL(String requestURL) { + this.requestURL = requestURL; + } + + public String getResponseContent() { + return responseContent; + } + + public void setResponseContent(String responseContent) { + this.responseContent = responseContent; + } + + public int getHttpCode() { + return httpCode; + } + + public void setHttpCode(int httpCode) { + this.httpCode = httpCode; + setMessage(); + } + + public void setMessage() { + message = buildMessage(httpCode); + } + + public void setMessage(String msg) { + + if(httpCode != -1) { + message = buildMessage(httpCode); + } else { + message = msg; + } + } + + public String getMessage(){ + return message; + } + + static String buildMessage(int code) { + + String message = ""; + + switch (code) { + + case 200: + message += "Request sucessful."; + break; + case 400: + message += "Could not properly intepret HTTP request."; + break; + case 401: + message += "User was not authorized. Try using credentials with access to all services. " + + "Ensure LDAP server is running."; + break; + case 403: + message += "Access to this resource is forbidden. It seems we might have made a bad request."; + break; + case 404: + message += "The page could not be found. Are the URLs for the topology services correct?"; + break; + case 500: + message += "The server encountered an error. Are all of the cluster's services running? \n" + + "Can a connection be established without Knox?"; + break; + + } + return message; + + } + + + } + + @XmlAccessorType(XmlAccessType.FIELD) + @XmlRootElement + public static class ServiceTestWrapper{ + + @XmlElement(name="ServiceTest") + @XmlElementWrapper(name="Tests") + private List<ServiceTest> tests = new ArrayList<ServiceTest>(); + + @XmlElement(name="message") + @XmlElementWrapper(name="messages") + private List<String> messages = new ArrayList<>(); + + public List<ServiceTest> getTests(){ + return tests; + } + + public void setTests(List<ServiceTest> st){ + this.tests = st; + } + + public List<String> getMessages() { + return messages; + } + + public void setMessages(List<String> messages){ + this.messages = messages; + } + + } + + +} http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestURL.java ---------------------------------------------------------------------- diff --git a/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestURL.java b/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestURL.java new file mode 100644 index 0000000..e50451a --- /dev/null +++ b/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestURL.java @@ -0,0 +1,89 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.gateway.service.test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ServiceTestURL { + + private static final Map<String, List<String>> urls; + static final String WEBHDFS = "WEBHDFS"; + static final String OOZIE = "OOZIE"; + static final String YARN = "RESOURCEMANAGER"; + static final String WEBHCAT = "WEBHCAT"; + static final String HBASE = "WEBHBASE"; + static final String HIVE = "HIVE"; + static final String STORM = "STORM"; + enum GATEWAY_SERVICES { + WEBHDFS, OOZIE, YARN, WEBHCAT, HBASE, HIVE, STORM + } + + static { + Map<String, List<String>> urlMap = new HashMap<>(); +// WEBHDFS + List<String> webhdfs = new ArrayList<>(); + webhdfs.add("/webhdfs/v1/?op=LISTSTATUS"); + urlMap.put(WEBHDFS, webhdfs); + +// OOZIE + List<String> oozie = new ArrayList<>(); + oozie.add("/oozie/v1/admin/build-version"); + oozie.add("/oozie/versions"); + oozie.add("/oozie/v1/admin/status"); + urlMap.put(OOZIE, oozie); + +// RESOURCEMANAGER + List<String> resourceManager = new ArrayList<>(); + resourceManager.add("/resourcemanager/v1/cluster/info"); + resourceManager.add("/resourcemanager/v1/cluster/metrics"); + resourceManager.add("/resourcemanager/v1/cluster/apps"); + urlMap.put(YARN, resourceManager); + +// WEBHCAT + List<String> templeton = new ArrayList<>(); + templeton.add("/templeton/v1/status"); + templeton.add("/templeton/v1/version"); + templeton.add("/templeton/v1/version/hive"); + templeton.add("/templeton/v1/version/hadoop"); + urlMap.put(WEBHCAT, templeton); + +// WEBHBASE + List<String> hbase = new ArrayList<>(); + hbase.add("/hbase/version"); + hbase.add("/hbase/version/cluster"); + hbase.add("/hbase/status/cluster"); + hbase.add("/hbase"); + urlMap.put(HBASE, hbase); + +// HIVE +// Not yet implemented +// ?????? + + urls = urlMap; + } + + + public static List<String> get(String url){ + return urls.get(url); + } + +} http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestWrapperMarshaller.java ---------------------------------------------------------------------- diff --git a/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestWrapperMarshaller.java b/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestWrapperMarshaller.java new file mode 100644 index 0000000..95f053d --- /dev/null +++ b/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/ServiceTestWrapperMarshaller.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.gateway.service.test; + +import org.eclipse.persistence.jaxb.JAXBContextProperties; + +import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.ext.MessageBodyWriter; +import javax.ws.rs.ext.Provider; +import javax.ws.rs.ext.Providers; +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import java.io.IOException; +import java.io.OutputStream; +import java.lang.annotation.Annotation; +import java.lang.reflect.Type; +import java.util.HashMap; +import java.util.Map; + +@Provider +@Produces({MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON}) +public class ServiceTestWrapperMarshaller implements MessageBodyWriter<ServiceTestResource.ServiceTestWrapper> { + + @Context + protected Providers providers; + + @Override + public boolean isWriteable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) { + return (ServiceTestResource.ServiceTestWrapper.class == type); + } + + @Override + public long getSize(ServiceTestResource.ServiceTestWrapper instance, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType) { + return -1; + } + + @Override + public void writeTo(ServiceTestResource.ServiceTestWrapper instance, Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, Object> httpHeaders, OutputStream entityStream) throws IOException, WebApplicationException { + try { + Map<String, Object> properties = new HashMap<String, Object>(1); + properties.put( JAXBContextProperties.MEDIA_TYPE, mediaType.toString()); + JAXBContext context = JAXBContext.newInstance(new Class[]{ServiceTestResource.ServiceTestWrapper.class}, properties); + Marshaller m = context.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + m.marshal(instance, entityStream); + + } catch (JAXBException e) { + throw new IOException(e); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/deploy/ServiceTestDeploymentContributor.java ---------------------------------------------------------------------- diff --git a/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/deploy/ServiceTestDeploymentContributor.java b/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/deploy/ServiceTestDeploymentContributor.java new file mode 100644 index 0000000..221c7a7 --- /dev/null +++ b/gateway-service-test/src/main/java/org/apache/hadoop/gateway/service/test/deploy/ServiceTestDeploymentContributor.java @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.gateway.service.test.deploy; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.gateway.deploy.DeploymentContext; +import org.apache.hadoop.gateway.descriptor.FilterParamDescriptor; +import org.apache.hadoop.gateway.descriptor.ResourceDescriptor; +import org.apache.hadoop.gateway.jersey.JerseyServiceDeploymentContributorBase; +import org.apache.hadoop.gateway.topology.Service; + +import java.util.ArrayList; +import java.util.List; + +/** + * + */ +public class ServiceTestDeploymentContributor extends JerseyServiceDeploymentContributorBase { + + private static final String PACKAGES_PARAM = "jersey.config.server.provider.packages"; + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.deploy.ServiceDeploymentContributor#getRole() + */ + @Override + public String getRole() { + return "SERVICE-TEST"; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.deploy.ServiceDeploymentContributor#getName() + */ + @Override + public String getName() { + return "service-test"; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.jersey.JerseyServiceDeploymentContributorBase#getPackages() + */ + @Override + protected String[] getPackages() { + return new String[]{ "org.apache.hadoop.gateway.service.test" }; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.gateway.jersey.JerseyServiceDeploymentContributorBase#getPatterns() + */ + @Override + protected String[] getPatterns() { + return new String[]{ "*/**?**", "/*" }; + } + + @Override + public void contributeService( DeploymentContext context, Service service ) throws Exception { + String packages = StringUtils.join(getPackages(), ";"); + for (String pattern : getPatterns()) { + ResourceDescriptor resource = context.getGatewayDescriptor().addResource(); + resource.role(service.getRole()); + resource.pattern(pattern); + addXForwardedFilter(context, service, resource); +// addAuthenticationFilter(context, service, resource); +// addIdentityAssertionFilter(context, service, resource); +// addAuthorizationFilter(context, service, resource); +// addRewriteFilter( context, service, resource, null ); + List<FilterParamDescriptor> params = new ArrayList<FilterParamDescriptor>(); + FilterParamDescriptor param = resource.createFilterParam(); + param.name(PACKAGES_PARAM); + param.value(packages); + params.add(param); + + FilterParamDescriptor traceType = resource.createFilterParam(); + traceType.name( "jersey.config.server.tracing" ); + traceType.value( "ALL" ); + params.add( traceType ); + FilterParamDescriptor traceLevel = resource.createFilterParam(); + traceLevel.name( "jersey.config.server.tracing.threshold" ); + traceLevel.value( "VERBOSE" ); + params.add( traceLevel ); + context.contributeFilter( service, resource, "pivot", "jersey", params ); + + + context.contributeFilter(service, resource, "pivot", "jersey", params); + + } + } +} http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-test/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ServiceDeploymentContributor ---------------------------------------------------------------------- diff --git a/gateway-service-test/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ServiceDeploymentContributor b/gateway-service-test/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ServiceDeploymentContributor new file mode 100644 index 0000000..cc1a096 --- /dev/null +++ b/gateway-service-test/src/main/resources/META-INF/services/org.apache.hadoop.gateway.deploy.ServiceDeploymentContributor @@ -0,0 +1,19 @@ +########################################################################## +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +########################################################################## + +org.apache.hadoop.gateway.service.test.deploy.ServiceTestDeploymentContributor \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-service-test/src/main/resources/org/apache/hadoop/gateway/service/test/jaxb.properties ---------------------------------------------------------------------- diff --git a/gateway-service-test/src/main/resources/org/apache/hadoop/gateway/service/test/jaxb.properties b/gateway-service-test/src/main/resources/org/apache/hadoop/gateway/service/test/jaxb.properties new file mode 100644 index 0000000..8c7ac2f --- /dev/null +++ b/gateway-service-test/src/main/resources/org/apache/hadoop/gateway/service/test/jaxb.properties @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +javax.xml.bind.context.factory=org.eclipse.persistence.jaxb.JAXBContextFactory \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-spi/src/main/java/org/apache/hadoop/gateway/services/topology/TopologyService.java ---------------------------------------------------------------------- diff --git a/gateway-spi/src/main/java/org/apache/hadoop/gateway/services/topology/TopologyService.java b/gateway-spi/src/main/java/org/apache/hadoop/gateway/services/topology/TopologyService.java index 9e787a7..a964f38 100644 --- a/gateway-spi/src/main/java/org/apache/hadoop/gateway/services/topology/TopologyService.java +++ b/gateway-spi/src/main/java/org/apache/hadoop/gateway/services/topology/TopologyService.java @@ -17,11 +17,14 @@ */ package org.apache.hadoop.gateway.services.topology; +import org.apache.hadoop.gateway.config.GatewayConfig; import org.apache.hadoop.gateway.services.Service; import org.apache.hadoop.gateway.topology.Topology; import org.apache.hadoop.gateway.topology.TopologyListener; import java.util.Collection; +import java.util.List; +import java.util.Map; public interface TopologyService extends Service { @@ -42,4 +45,6 @@ public interface TopologyService extends Service { public void deleteTopology(Topology t); -} + public Map<String, List<String>> getServiceTestURLs(Topology t, GatewayConfig config); + + } http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java ---------------------------------------------------------------------- diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java index 8e2bfe4..51add41 100644 --- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java +++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayBasicFuncTest.java @@ -19,11 +19,13 @@ package org.apache.hadoop.gateway; import com.jayway.restassured.http.ContentType; import com.jayway.restassured.response.Cookie; +import com.jayway.restassured.response.Header; import com.jayway.restassured.response.Response; import com.jayway.restassured.specification.ResponseSpecification; import com.mycila.xmltool.XMLDoc; import com.mycila.xmltool.XMLTag; import org.apache.commons.io.filefilter.WildcardFileFilter; +import org.apache.hadoop.gateway.util.KnoxCLI; import org.apache.hadoop.test.TestUtils; import org.apache.hadoop.test.category.FunctionalTests; import org.apache.hadoop.test.category.MediumTests; @@ -49,11 +51,14 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.FileFilter; -import java.io.FileNotFoundException; +import javax.ws.rs.core.MediaType; +import java.io.PrintStream; import java.io.IOException; import java.io.StringWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileFilter; +import java.io.ByteArrayOutputStream; import java.net.InetAddress; import java.net.InetSocketAddress; import java.net.ServerSocket; @@ -66,6 +71,7 @@ import java.util.Map.Entry; import static com.jayway.restassured.RestAssured.given; import static org.hamcrest.CoreMatchers.*; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.text.IsEmptyString.isEmptyString; import static org.xmlmatchers.XmlMatchers.isEquivalentTo; @@ -230,17 +236,19 @@ public class GatewayBasicFuncTest { .addTag( "role" ).addText( "WEBHBASE" ) .addTag( "url" ).addText( driver.getRealUrl( "WEBHBASE" ) ).gotoParent() .addTag("service") - .addTag( "role" ).addText( "RESOURCEMANAGER" ) - .addTag( "url" ).addText( driver.getRealUrl( "RESOURCEMANAGER" ) ).gotoParent() - .addTag( "service" ) - .addTag( "role" ).addText( "FALCON" ) - .addTag( "url" ).addText( driver.getRealUrl( "FALCON" ) ).gotoParent() - .addTag( "service" ) - .addTag( "role" ).addText( "STORM" ) - .addTag( "url" ).addText( driver.getRealUrl( "STORM" ) ).gotoParent() - .addTag( "service" ) - .addTag( "role" ).addText( "STORM-LOGVIEWER" ) - .addTag( "url" ).addText( driver.getRealUrl( "STORM-LOGVIEWER" ) ).gotoParent() + .addTag("role").addText("RESOURCEMANAGER") + .addTag("url").addText(driver.getRealUrl("RESOURCEMANAGER")).gotoParent() + .addTag("service") + .addTag("role").addText("FALCON") + .addTag("url").addText(driver.getRealUrl("FALCON")).gotoParent() + .addTag("service") + .addTag("role").addText("STORM") + .addTag("url").addText(driver.getRealUrl("STORM")).gotoParent() + .addTag("service") + .addTag("role").addText("STORM-LOGVIEWER") + .addTag("url").addText(driver.getRealUrl("STORM-LOGVIEWER")).gotoParent() + .addTag("service") + .addTag("role").addText("SERVICE-TEST") .gotoRoot(); // System.out.println( "GATEWAY=" + xml.toString() ); return xml; @@ -3411,4 +3419,214 @@ public class GatewayBasicFuncTest { MatcherAssert.assertThat(location, not(containsString("host="))); MatcherAssert.assertThat(location, not(containsString("port="))); } + + @Test + public void testServiceTestAPI() throws Exception { + + String user = "kminder"; + String password = "kminder-password"; + + String queryString = "?username=" + user + "&password=" + password; + + String clusterUrl = driver.getClusterUrl(); + String testUrl = clusterUrl + "/service-test"; + +// XML Response + setupResources(); + given() + .header(new Header("Accept", MediaType.APPLICATION_XML)) + .expect() + .contentType(MediaType.APPLICATION_XML) + .statusCode(HttpStatus.SC_OK) + .body(not(containsString("<httpCode>401"))) + .body(not(containsString("<httpCode>404"))) + .body(not(containsString("<httpCode>403"))) + .body(containsString("<httpCode>200")) + .when() + .get(testUrl + queryString); +// .prettyPrint(); + driver.assertComplete(); + +// JSON Response + setupResources(); + given() + .header(new Header("Accept", MediaType.APPLICATION_JSON)) + .expect() + .statusCode(HttpStatus.SC_OK) + .contentType(MediaType.APPLICATION_JSON) + .body(not(containsString("\"httpCode\" : 401"))) + .body(not(containsString("\"httpCode\" : 404"))) + .body(not(containsString("\"httpCode\" : 403"))) + .body(containsString("\"httpCode\" : 200")) + .when() + .get(testUrl + queryString); +// .prettyPrint(); + driver.assertComplete(); + +// Test authorization with a header instead + setupResources(); + given() + .header(new Header("Accept", MediaType.APPLICATION_JSON)) + .auth().preemptive().basic("kminder", "kminder-password") + .expect() + .statusCode(HttpStatus.SC_OK) + .contentType(MediaType.APPLICATION_JSON) + .body(not(containsString("\"httpCode\" : 401"))) + .body(not(containsString("\"httpCode\" : 404"))) + .body(not(containsString("\"httpCode\" : 403"))) + .body(containsString("\"httpCode\" : 200")) + .when() + .get(testUrl); +// .prettyPrint(); + driver.assertComplete(); + + + +// Authorize as a different (invalid) user + setupResources(); + given() + .header(new Header("Accept", MediaType.APPLICATION_JSON)) + .expect() + .statusCode(HttpStatus.SC_OK) + .contentType(MediaType.APPLICATION_JSON) + .body(not(containsString("\"httpCode\" : 200"))) + .body(not(containsString("\"httpCode\" : 404"))) + .body(not(containsString("\"httpCode\" : 403"))) + .body(containsString("\"httpCode\" : 401")) + .when() + .get(testUrl + "?username=bad-user&password=bad-password"); +// .prettyPrint(); + driver.assertNotComplete("WEBHDFS"); + driver.assertNotComplete("OOZIE"); + driver.assertNotComplete("RESOURCEMANAGER"); + driver.assertNotComplete("WEBHCAT"); + driver.assertNotComplete("STORM"); + driver.assertNotComplete("WEBHBASE"); + driver.assertNotComplete("FALCON"); + + // Authorize as a different (valid) user + setupResources(); + given() + .header(new Header("Accept", MediaType.APPLICATION_JSON)) + .expect() + .statusCode(HttpStatus.SC_OK) + .contentType(MediaType.APPLICATION_JSON) + .body(not(containsString("\"httpCode\" : 401"))) + .body(not(containsString("\"httpCode\" : 404"))) + .body(not(containsString("\"httpCode\" : 403"))) + .when() + .get(testUrl + "?username=mapred&password=mapred-password"); +// .prettyPrint(); + driver.assertComplete(); + } + + @Test + public void testCLIServiceTest() throws Exception { + + setupResources(); + // Now let's make sure we can run this same command from the CLI. + PrintStream out = System.out; + InetSocketAddress gatewayAddress = driver.gateway.getAddresses()[0]; + final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + System.setOut(new PrintStream(outContent)); + + String args[] = {"service-test", "--master", "knox", "--cluster", driver.clusterName, "--hostname", gatewayAddress.getHostName(), + "--port", Integer.toString(gatewayAddress.getPort()), "--u", "kminder","--p", "kminder-password" }; + KnoxCLI cli = new KnoxCLI(); + cli.run(args); + + assertThat(outContent.toString(), not(containsString("\"httpCode\": 401"))); + assertThat( outContent.toString(), not(containsString("404"))); + assertThat(outContent.toString(), not(containsString("403"))); + outContent.reset(); + + setupResources(); + + + String args2[] = {"service-test", "--master", "knox", "--cluster", driver.clusterName, "--hostname", gatewayAddress.getHostName(), + "--port", Integer.toString(gatewayAddress.getPort()) }; + + cli = new KnoxCLI(); + cli.run(args2); + assertThat(outContent.toString(), (containsString("Username and/or password not supplied. Expect HTTP 401 Unauthorized responses."))); + outContent.reset(); + + + String args3[] = {"service-test", "--master", "knox", "--cluster", driver.clusterName, "--hostname", "bad-host", + "--port", "0" }; + + cli = new KnoxCLI(); + cli.run(args3); + assertThat(outContent.toString(), containsString("nodename nor servname provided")); + outContent.reset(); + + String args4[] = {"service-test", "--master", "knox", "--cluster", driver.clusterName, "--hostname", gatewayAddress.getHostName(), + "--port", "543", "--u", "mapred", "--p", "mapred-password" }; + + cli = new KnoxCLI(); + cli.run(args4); + assertThat(outContent.toString(), containsString("failed: Connection refused")); + outContent.reset(); + + + String args5[] = {"service-test", "--master", "knox", "--hostname", gatewayAddress.getHostName(), + "--port", "543", "--u", "mapred", "--p", "mapred-password" }; + + cli = new KnoxCLI(); + cli.run(args5); + assertThat(outContent.toString(), containsString("--cluster argument is required")); + outContent.reset(); + +// Reset the out content + System.setOut(out); + } + + + void setupResource(String serviceRole, String path){ + driver.getMock(serviceRole) + .expect().method("GET") + .pathInfo(path) + .respond() + .status(HttpStatus.SC_OK) + .contentType("application/json") + .characterEncoding("utf-8"); +// .content(driver.getResourceBytes(classLoaderResource + "." + type.toString().toLowerCase())) +// .contentType(type.toString()); + } + + void setupResources() { + + driver.setResourceBase(GatewayBasicFuncTest.class); + + try { + setupResource("WEBHDFS", "/v1/"); + setupResource("WEBHCAT", "/v1/status"); + setupResource("WEBHCAT", "/v1/version"); + setupResource("WEBHCAT", "/v1/version/hive"); + setupResource("WEBHCAT", "/v1/version/hadoop"); + setupResource("OOZIE", "/v1/admin/build-version"); + setupResource("OOZIE", "/v1/admin/status"); + setupResource("OOZIE", "/versions"); + setupResource("WEBHBASE", "/version"); + setupResource("WEBHBASE", "/version/cluster"); + setupResource("WEBHBASE", "/status/cluster"); + setupResource("WEBHBASE", "/"); + setupResource("RESOURCEMANAGER", "/v1/cluster/info/"); + setupResource("RESOURCEMANAGER", "/v1/cluster/metrics/"); + setupResource("RESOURCEMANAGER", "/v1/cluster/apps/"); + setupResource("STORM", "/api/v1/cluster/configuration"); + setupResource("STORM", "/api/v1/cluster/summary"); + setupResource("STORM", "/api/v1/supervisor/summary"); + setupResource("STORM", "/api/v1/topology/summary"); + setupResource("FALCON", "/api/admin/stack"); + setupResource("FALCON", "/api/admin/version"); + setupResource("FALCON", "/api/metadata/lineage/serialize"); + setupResource("FALCON", "/api/metadata/lineage/vertices/all"); + setupResource("FALCON", "/api/metadata/lineage/edges/all"); + + } catch (Exception e) { + System.out.println(e.getMessage()); + } + } + } http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java ---------------------------------------------------------------------- diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java index 2d6c4be..b4965c8 100644 --- a/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java +++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/GatewayFuncTestDriver.java @@ -70,7 +70,9 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.isIn; +import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertThat; +import static org.junit.Assert.fail; /** * This class was created to reduce much of the duplication and boiler plate that was ending up in the GatewayBasicFuncTest class. @@ -91,6 +93,7 @@ public class GatewayFuncTestDriver { public boolean useGateway; public GatewayServer gateway; public GatewayConfig config; + public String clusterName; /** * Sets the class from which relative test resource names should be resolved. @@ -130,6 +133,7 @@ public class GatewayFuncTestDriver { public void setupGateway( GatewayTestConfig config, String cluster, XMLTag topology, boolean use ) throws Exception { this.useGateway = use; this.config = config; + this.clusterName = cluster; File targetDir = new File( System.getProperty( "user.dir" ), "target" ); File gatewayDir = new File( targetDir, "gateway-home-" + UUID.randomUUID() ); @@ -233,6 +237,13 @@ public class GatewayFuncTestDriver { return url; } + public String getClusterUrl() { + String url; + String localHostName = getLocalHostName(); + url = "http://" + localHostName + ":" + gateway.getAddresses()[0].getPort() + "/" + config.getGatewayPath() + "/" + clusterName; + return url; + } + public String getRealAddr( String role ) { String addr; String localHostName = getLocalHostName(); @@ -314,6 +325,25 @@ public class GatewayFuncTestDriver { } } + + public void assertNotComplete(String serviceName) { + // Check to make sure that all interaction were satisfied if for mocked services. + // Otherwise just clear the mock interaction queue. + + Service service = services.get(serviceName); + + if(service != null) { + if(service.mock) { + assertThat( + "Service " + service.role + " has remaining expected interactions.", + service.server.getCount(), not(0)); + } + service.server.reset(); + } else { + fail(); + } + } + public void reset() { for( Service service : services.values() ) { service.server.reset(); http://git-wip-us.apache.org/repos/asf/knox/blob/fae40583/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 4c1f236..cc66fab 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,7 @@ <module>gateway-test</module> <module>hsso-release</module> <module>gateway-service-vault</module> + <module>gateway-service-test</module> </modules> <properties> @@ -498,6 +499,11 @@ </dependency> <dependency> <groupId>${gateway-group}</groupId> + <artifactId>gateway-service-test</artifactId> + <version>${gateway-version}</version> + </dependency> + <dependency> + <groupId>${gateway-group}</groupId> <artifactId>gateway-service-as</artifactId> <version>${gateway-version}</version> </dependency>