KNOX-483 added webhdfs sans ha

Project: http://git-wip-us.apache.org/repos/asf/knox/repo
Commit: http://git-wip-us.apache.org/repos/asf/knox/commit/e9fe7bcc
Tree: http://git-wip-us.apache.org/repos/asf/knox/tree/e9fe7bcc
Diff: http://git-wip-us.apache.org/repos/asf/knox/diff/e9fe7bcc

Branch: refs/heads/KNOX-481
Commit: e9fe7bccf04bf1ac61e50a9c3d08359efbee2a29
Parents: 7258c49
Author: Sumit Gupta <su...@apache.org>
Authored: Mon Jan 19 12:21:03 2015 -0500
Committer: Sumit Gupta <su...@apache.org>
Committed: Wed Feb 11 13:24:58 2015 -0500

----------------------------------------------------------------------
 .../services/webhdfs/2.4.0/rewrite.xml          | 70 ++++++++++++++++++++
 .../services/webhdfs/2.4.0/service.xml          | 38 +++++++++++
 .../services/yarn-rm/2.5.0/service.xml          |  2 -
 .../hdfs/WebHdfsDeploymentContributor.java      | 10 +--
 .../deploy/DeploymentFactoryFuncTest.java       | 34 +++++++++-
 5 files changed, 145 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
----------------------------------------------------------------------
diff --git 
a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
 
b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
new file mode 100644
index 0000000..efbd93d
--- /dev/null
+++ 
b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/rewrite.xml
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<rules>
+
+    <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" 
pattern="hdfs://*:*/{path=**}?{**}">
+        <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="OUT" name="WEBHDFS/webhdfs/outbound" 
pattern="webhdfs://*:*/{path=**}?{**}">
+        <rewrite template="{$frontend[url]}/webhdfs/v1/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="OUT" name="WEBHDFS/webhdfs/outbound/namenode/headers/location">
+        <match pattern="{scheme}://{host}:{port}/{path=**}?{**}"/>
+        <rewrite 
template="{$frontend[url]}/webhdfs/data/v1/{path=**}?{scheme}?host={$hostmap(host)}?{port}?{**}"/>
+        <encrypt-query/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/hdfs" 
pattern="hdfs:/{path=**}?{**}">
+        <rewrite template="{$serviceMappedUrl[NAMENODE]}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/webhdfs" 
pattern="webhdfs:/{path=**}?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/root" 
pattern="*://*:*/**/webhdfs/{version}/?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/file" 
pattern="*://*:*/**/webhdfs/{version}/{path=**}?{**}">
+        <rewrite template="{$serviceUrl[WEBHDFS]}/{version}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home" 
pattern="*://*:*/**/webhdfs/{version}/~?{**}">
+        <rewrite 
template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/namenode/home/file" 
pattern="*://*:*/**/webhdfs/{version}/~/{path=**}?{**}">
+        <rewrite 
template="{$serviceUrl[WEBHDFS]}/{version}/user/{$username}/{path=**}?{**}"/>
+    </rule>
+
+    <rule dir="IN" name="WEBHDFS/webhdfs/inbound/datanode">
+        <decrypt-query/>
+        <match 
pattern="*://*:*/**/webhdfs/data/*/{path=**}?{scheme}?{host}?{port}?{**}"/>
+        <rewrite template="{scheme}://{host}:{port}/{path=**}?{**}"/>
+    </rule>
+
+    <filter name="WEBHDFS/webhdfs/outbound/namenode/headers">
+        <content type="application/x-http-headers">
+            <apply path="Location" 
rule="WEBHDFS/webhdfs/outbound/namenode/headers/location"/>
+        </content>
+    </filter>
+
+</rules>
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
----------------------------------------------------------------------
diff --git 
a/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
 
b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
new file mode 100644
index 0000000..1aeaf70
--- /dev/null
+++ 
b/gateway-service-definitions/src/main/resources/services/webhdfs/2.4.0/service.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<service role="WEBHDFS" name="webhdfs" version="2.4.0">
+    <urls>
+        <url pattern="/webhdfs/v1/?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/root" 
apply-to="request.url"/>
+        </url>
+        <url pattern="/webhdfs/v1/**?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/file" 
apply-to="request.url"/>
+            <rewrite-filter ref="WEBHDFS/webhdfs/outbound/namenode/headers" 
apply-to="response.headers"/>
+        </url>
+        <url pattern="/webhdfs/v1/~?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/home" 
apply-to="request.url"/>
+        </url>
+        <url pattern="/webhdfs/v1/~/**?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/namenode/home/file" 
apply-to="request.url"/>
+            <rewrite-filter ref="WEBHDFS/webhdfs/outbound/namenode/headers" 
apply-to="response.headers"/>
+        </url>
+        <url pattern="/webhdfs/data/v1/**?**">
+            <rewrite-filter ref="WEBHDFS/webhdfs/inbound/datanode" 
apply-to="request.url"/>
+        </url>
+    </urls>
+</service>

http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
----------------------------------------------------------------------
diff --git 
a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
 
b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
index 61a77f0..f77d94d 100644
--- 
a/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
+++ 
b/gateway-service-definitions/src/main/resources/services/yarn-rm/2.5.0/service.xml
@@ -16,13 +16,11 @@
    limitations under the License.
 -->
 <service role="RESOURCEMANAGER" name="resourcemanager" version="2.5.0">
-    <policies></policies>
     <urls>
         <url pattern="/resourcemanager/v1/cluster/"/>
         <url pattern="/resourcemanager/v1/cluster/**?**"/>
         <url pattern="/resourcemanager/v1/cluster/apps?**">
             <rewrite-filter 
ref="RESOURCEMANAGER/resourcemanager/apps/outbound" apply-to="response.body"/>
-            <policies></policies>
         </url>
         <url pattern="/resourcemanager/v1/cluster/apps?**">
             <rewrite-filter 
ref="RESOURCEMANAGER/resourcemanager/apps/outbound" apply-to="response.body"/>

http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
----------------------------------------------------------------------
diff --git 
a/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
 
b/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
index 0e63597..295e081 100644
--- 
a/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
+++ 
b/gateway-service-webhdfs/src/main/java/org/apache/hadoop/gateway/hdfs/WebHdfsDeploymentContributor.java
@@ -44,19 +44,19 @@ public class WebHdfsDeploymentContributor extends 
ServiceDeploymentContributorBa
 
   @Override
   public String getRole() {
-    return "WEBHDFS";
+    return "xWEBHDFS";
   }
 
   @Override
   public String getName() {
-    return "webhdfs";
+    return "xwebhdfs";
   }
 
   @Override
   public void contributeService( DeploymentContext context, Service service ) 
throws Exception {
-    contributeRewriteRules( context, service );
-    contributeNameNodeResource( context, service );
-    contributeDataNodeResource( context, service );
+//    contributeRewriteRules( context, service );
+//    contributeNameNodeResource( context, service );
+//    contributeDataNodeResource( context, service );
   }
 
   private void contributeRewriteRules( DeploymentContext context, Service 
service ) throws URISyntaxException, IOException {

http://git-wip-us.apache.org/repos/asf/knox/blob/e9fe7bcc/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
----------------------------------------------------------------------
diff --git 
a/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
 
b/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
index d8b1893..c59dd49 100644
--- 
a/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
+++ 
b/gateway-test/src/test/java/org/apache/hadoop/gateway/deploy/DeploymentFactoryFuncTest.java
@@ -82,6 +82,7 @@ public class DeploymentFactoryFuncTest {
 
 //    ((GatewayTestConfig) config).setDeploymentDir( "clusters" );
 
+    addStacksDir(config, gatewayDir);
     DefaultGatewayServices srvcs = new DefaultGatewayServices();
     Map<String,String> options = new HashMap<String,String>();
     options.put("persist-master", "false");
@@ -135,6 +136,7 @@ public class DeploymentFactoryFuncTest {
     ((GatewayTestConfig) config).setGatewayHomeDir( 
gatewayDir.getAbsolutePath() );
     File deployDir = new File( config.getGatewayDeploymentDir() );
     deployDir.mkdirs();
+    addStacksDir(config, gatewayDir);
 
     DefaultGatewayServices srvcs = new DefaultGatewayServices();
     Map<String,String> options = new HashMap<String,String>();
@@ -189,6 +191,7 @@ public class DeploymentFactoryFuncTest {
     ((GatewayTestConfig) config).setGatewayHomeDir( 
gatewayDir.getAbsolutePath() );
     File deployDir = new File( config.getGatewayDeploymentDir() );
     deployDir.mkdirs();
+    addStacksDir(config, gatewayDir);
 
     DefaultGatewayServices srvcs = new DefaultGatewayServices();
     Map<String,String> options = new HashMap<String,String>();
@@ -228,8 +231,8 @@ public class DeploymentFactoryFuncTest {
     topology.addProvider( authorizer );
 
     WebArchive war = DeploymentFactory.createDeployment( config, topology );
-    //File dir = new File( System.getProperty( "user.dir" ) );
-    //File file = war.as( ExplodedExporter.class ).exportExploded( dir, 
"test-cluster.war" );
+//    File dir = new File( System.getProperty( "user.dir" ) );
+//    File file = war.as( ExplodedExporter.class ).exportExploded( dir, 
"test-cluster.war" );
 
     Document web = parse( war.get( "WEB-INF/web.xml" ).getAsset().openStream() 
);
     assertThat( web, hasXPath( "/web-app/servlet/servlet-name", equalTo( 
"test-cluster" ) ) );
@@ -538,6 +541,33 @@ public class DeploymentFactoryFuncTest {
     return builder.parse( source );
   }
 
+  private void addStacksDir(GatewayConfig config, File targetDir) {
+    File stacksDir = new File( config.getGatewayStacksDir() );
+    stacksDir.mkdirs();
+    //TODO: [sumit] This is a hack for now, need to find a better way to 
locate the source resources for 'stacks' to be tested
+    String pathToStacksSource = 
"gateway-service-definitions/src/main/resources/services";
+    File stacksSourceDir = new File( targetDir.getParent(), 
pathToStacksSource);
+    if (!stacksSourceDir.exists()) {
+      stacksSourceDir = new File( targetDir.getParentFile().getParent(), 
pathToStacksSource);
+    }
+    if (stacksSourceDir.exists()) {
+      try {
+        FileUtils.copyDirectoryToDirectory(stacksSourceDir, stacksDir);
+      } catch ( IOException e) {
+        fail(e.getMessage());
+      }
+    }
+
+  }
+//  private void dump( Document document ) throws TransformerException {
+//    Transformer transformer = 
TransformerFactory.newInstance().newTransformer();
+//    transformer.setOutputProperty( OutputKeys.INDENT, "yes" );
+//    StreamResult result = new StreamResult( new StringWriter() );
+//    DOMSource source = new DOMSource( document );
+//    transformer.transform( source, result );
+//    String xmlString = result.getWriter().toString();
+//    System.out.println( xmlString );
+//  }
   private void dump( Document document ) throws TransformerException {
     Transformer transformer = 
TransformerFactory.newInstance().newTransformer();
     transformer.setOutputProperty( OutputKeys.INDENT, "yes" );

Reply via email to