KNOX-483 added webhdfs sans ha
Project: http://git-wip-us.apache.org/repos/asf/knox/repo Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/e9fe7bcc Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/e9fe7bcc Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/e9fe7bcc Branch: refs/heads/KNOX-481 Commit: e9fe7bccf04bf1ac61e50a9c3d08359efbee2a29 Parents: 7258c49 Author: Sumit Gupta <su...@apache.org> Authored: Mon Jan 19 12:21:03 2015 -0500 Committer: Sumit Gupta <su...@apache.org> Committed: Wed Feb 11 13:24:58 2015 -0500 ---------------------------------------------------------------------- .../services/webhdfs/2.4.0/rewrite.xml | 70 ++++++++++++++++++++ .../services/webhdfs/2.4.0/service.xml | 38 +++++++++++ .../services/yarn-rm/2.5.0/service.xml | 2 - .../hdfs/WebHdfsDeploymentContributor.java | 10 +-- .../deploy/DeploymentFactoryFuncTest.java | 34 +++++++++- 5 files changed, 145 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml new file mode 100644 index 0000000..efbd93d --- /dev/null +++ b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml @@ -0,0 +1,70 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<rules> + + <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" pattern="hdfs://*:*/{path=**}?{**}"> + <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/> + </rule> + + <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" pattern="webhdfs://*:*/{path=**}?{**}"> + <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/> + </rule> + + <rule dir="OUT" name="WEBHDFS/webhdfs/outbound/namenode/headers/location"> + <match pattern="{scheme}://{host}:{port}/{path=**}?{**}"/> + <rewrite template="{$frontend[url]}/webhdfs/data/v1/{path=**}?{scheme}?host={$hostmap(host)}?{port}?{**}"/> + <encrypt-query/> + </rule> + + <rule dir="IN" name="WEBHDFS/webhdfs/inbound/hdfs" pattern="hdfs:/{path=**}?{**}"> + <rewrite template="{$serviceMappedUrl[NAMENODE]}/{path=**}?{**}"/> + </rule> + + <rule dir="IN" name="WEBHDFS/webhdfs/inbound/webhdfs" pattern="webhdfs:/{path=**}?{**}"> + <rewrite template="{$serviceUrl[WEBHDFS]}/{path=**}?{**}"/> + </rule> + + <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/root" pattern="*://*:*/**/webhdfs/{version}/?{**}"> + <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/?{**}"/> + </rule> + + <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/file" pattern="*://*:*/**/webhdfs/{version}/{path=**}?{**}"> + <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/{path=**}?{**}"/> + </rule> + + <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home" pattern="*://*:*/**/webhdfs/{version}/~?{**}"> + <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}?{**}"/> + </rule> + + <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home/file" pattern="*://*:*/**/webhdfs/{version}/~/{path=**}?{**}"> + <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}/{path=**}?{**}"/> + </rule> + + <rule dir="IN" name="WEBHDFS/webhdfs/inbound/datanode"> + <decrypt-query/> + <match pattern="*://*:*/**/webhdfs/data/*/{path=**}?{scheme}?{host}?{port}?{**}"/> + <rewrite template="{scheme}://{host}:{port}/{path=**}?{**}"/> + </rule> + + <filter name="WEBHDFS/webhdfs/outbound/namenode/headers"> + <content type="application/x-http-headers"> + <apply path="Location" rule="WEBHDFS/webhdfs/outbound/namenode/headers/location"/> + </content> + </filter> + +</rules> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml new file mode 100644 index 0000000..1aeaf70 --- /dev/null +++ b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml @@ -0,0 +1,38 @@ +<?xml version="1.0" encoding="UTF-8" standalone="yes"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<service role="WEBHDFS" name="webhdfs" version="2.4.0"> + <urls> + <url pattern="/webhdfs/v1/?**"> + <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/root" apply-to="request.url"/> + </url> + <url pattern="/webhdfs/v1/**?**"> + <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/file" apply-to="request.url"/> + <rewrite-filter ref="WEBHDFS/webhdfs/outbound/namenode/headers" apply-to="response.headers"/> + </url> + <url pattern="/webhdfs/v1/~?**"> + <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/home" apply-to="request.url"/> + </url> + <url pattern="/webhdfs/v1/~/**?**"> + <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/home/file" apply-to="request.url"/> + <rewrite-filter ref="WEBHDFS/webhdfs/outbound/namenode/headers" apply-to="response.headers"/> + </url> + <url pattern="/webhdfs/data/v1/**?**"> + <rewrite-filter ref="WEBHDFS/webhdfs/inbound/datanode" apply-to="request.url"/> + </url> + </urls> +</service> http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml ---------------------------------------------------------------------- diff --git a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml index 61a77f0..f77d94d 100644 --- a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml +++ b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml @@ -16,13 +16,11 @@ limitations under the License. --> <service role="RESOURCEMANAGER" name="resourcemanager" version="2.5.0"> - <policies></policies> <urls> <url pattern="/resourcemanager/v1/cluster/"/> <url pattern="/resourcemanager/v1/cluster/**?**"/> <url pattern="/resourcemanager/v1/cluster/apps?**"> <rewrite-filter ref="RESOURCEMANAGER/resourcemanager/apps/outbound" apply-to="response.body"/> - <policies></policies> </url> <url pattern="/resourcemanager/v1/cluster/apps?**"> <rewrite-filter ref="RESOURCEMANAGER/resourcemanager/apps/outbound" apply-to="response.body"/> http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java ---------------------------------------------------------------------- diff --git a/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java b/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java index 0e63597..295e081 100644 --- a/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java +++ b/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java @@ -44,19 +44,19 @@ public class WebHdfsDeploymentContributor extends ServiceDeploymentContributorBa @Override public String getRole() { - return "WEBHDFS"; + return "xWEBHDFS"; } @Override public String getName() { - return "webhdfs"; + return "xwebhdfs"; } @Override public void contributeService( DeploymentContext context, Service service ) throws Exception { - contributeRewriteRules( context, service ); - contributeNameNodeResource( context, service ); - contributeDataNodeResource( context, service ); +// contributeRewriteRules( context, service ); +// contributeNameNodeResource( context, service ); +// contributeDataNodeResource( context, service ); } private void contributeRewriteRules( DeploymentContext context, Service service ) throws URISyntaxException, IOException { http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java ---------------------------------------------------------------------- diff --git a/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java b/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java index d8b1893..c59dd49 100644 --- a/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java +++ b/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java @@ -82,6 +82,7 @@ public class DeploymentFactoryFuncTest { // ((GatewayTestConfig) config).setDeploymentDir( "clusters" ); + addStacksDir(config, gatewayDir); DefaultGatewayServices srvcs = new DefaultGatewayServices(); Map<String,String> options = new HashMap<String,String>(); options.put("persist-master", "false"); @@ -135,6 +136,7 @@ public class DeploymentFactoryFuncTest { ((GatewayTestConfig) config).setGatewayHomeDir( gatewayDir.getAbsolutePath() ); File deployDir = new File( config.getGatewayDeploymentDir() ); deployDir.mkdirs(); + addStacksDir(config, gatewayDir); DefaultGatewayServices srvcs = new DefaultGatewayServices(); Map<String,String> options = new HashMap<String,String>(); @@ -189,6 +191,7 @@ public class DeploymentFactoryFuncTest { ((GatewayTestConfig) config).setGatewayHomeDir( gatewayDir.getAbsolutePath() ); File deployDir = new File( config.getGatewayDeploymentDir() ); deployDir.mkdirs(); + addStacksDir(config, gatewayDir); DefaultGatewayServices srvcs = new DefaultGatewayServices(); Map<String,String> options = new HashMap<String,String>(); @@ -228,8 +231,8 @@ public class DeploymentFactoryFuncTest { topology.addProvider( authorizer ); WebArchive war = DeploymentFactory.createDeployment( config, topology ); - //File dir = new File( System.getProperty( "user.dir" ) ); - //File file = war.as( ExplodedExporter.class ).exportExploded( dir, "test-cluster.war" ); +// File dir = new File( System.getProperty( "user.dir" ) ); +// File file = war.as( ExplodedExporter.class ).exportExploded( dir, "test-cluster.war" ); Document web = parse( war.get( "WEB-INF/web.xml" ).getAsset().openStream() ); assertThat( web, hasXPath( "/web-app/servlet/servlet-name", equalTo( "test-cluster" ) ) ); @@ -538,6 +541,33 @@ public class DeploymentFactoryFuncTest { return builder.parse( source ); } + private void addStacksDir(GatewayConfig config, File targetDir) { + File stacksDir = new File( config.getGatewayStacksDir() ); + stacksDir.mkdirs(); + //TODO: [sumit] This is a hack for now, need to find a better way to locate the source resources for 'stacks' to be tested + String pathToStacksSource = "gateway-service-definitions/src/main/resources/services"; + File stacksSourceDir = new File( targetDir.getParent(), pathToStacksSource); + if (!stacksSourceDir.exists()) { + stacksSourceDir = new File( targetDir.getParentFile().getParent(), pathToStacksSource); + } + if (stacksSourceDir.exists()) { + try { + FileUtils.copyDirectoryToDirectory(stacksSourceDir, stacksDir); + } catch ( IOException e) { + fail(e.getMessage()); + } + } + + } +// private void dump( Document document ) throws TransformerException { +// Transformer transformer = TransformerFactory.newInstance().newTransformer(); +// transformer.setOutputProperty( OutputKeys.INDENT, "yes" ); +// StreamResult result = new StreamResult( new StringWriter() ); +// DOMSource source = new DOMSource( document ); +// transformer.transform( source, result ); +// String xmlString = result.getWriter().toString(); +// System.out.println( xmlString ); +// } private void dump( Document document ) throws TransformerException { Transformer transformer = TransformerFactory.newInstance().newTransformer(); transformer.setOutputProperty( OutputKeys.INDENT, "yes" );
