Author: tucu Date: Fri Feb 1 19:46:14 2013 New Revision: 1441606 URL: http://svn.apache.org/viewvc?rev=1441606&view=rev Log: HDFS-4456. Add concat to HttpFS and WebHDFS REST API docs. (plamenj2003 via tucu)
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java?rev=1441606&r1=1441605&r2=1441606&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/client/HttpFSFileSystem.java Fri Feb 1 19:46:14 2013 @@ -17,6 +17,8 @@ */ package org.apache.hadoop.fs.http.client; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; @@ -86,6 +88,7 @@ public class HttpFSFileSystem extends Fi public static final String PERMISSION_PARAM = "permission"; public static final String DESTINATION_PARAM = "destination"; public static final String RECURSIVE_PARAM = "recursive"; + public static final String SOURCES_PARAM = "sources"; public static final String OWNER_PARAM = "owner"; public static final String GROUP_PARAM = "group"; public static final String MODIFICATION_TIME_PARAM = "modificationtime"; @@ -167,7 +170,7 @@ public class HttpFSFileSystem extends Fi GETHOMEDIRECTORY(HTTP_GET), GETCONTENTSUMMARY(HTTP_GET), GETFILECHECKSUM(HTTP_GET), GETFILEBLOCKLOCATIONS(HTTP_GET), INSTRUMENTATION(HTTP_GET), - APPEND(HTTP_POST), + APPEND(HTTP_POST), CONCAT(HTTP_POST), CREATE(HTTP_PUT), MKDIRS(HTTP_PUT), RENAME(HTTP_PUT), SETOWNER(HTTP_PUT), SETPERMISSION(HTTP_PUT), SETREPLICATION(HTTP_PUT), SETTIMES(HTTP_PUT), DELETE(HTTP_DELETE); @@ -529,6 +532,29 @@ public class HttpFSFileSystem extends Fi } /** + * Concat existing files together. + * @param f the path to the target destination. + * @param psrcs the paths to the sources to use for the concatenation. + * + * @throws IOException + */ + @Override + public void concat(Path f, Path[] psrcs) throws IOException { + List<String> strPaths = new ArrayList<String>(psrcs.length); + for(Path psrc : psrcs) { + strPaths.add(psrc.toUri().getPath()); + } + String srcs = StringUtils.join(",", strPaths); + + Map<String, String> params = new HashMap<String, String>(); + params.put(OP_PARAM, Operation.CONCAT.toString()); + params.put(SOURCES_PARAM, srcs); + HttpURLConnection conn = getConnection(Operation.CONCAT.getMethod(), + params, f, true); + HttpFSUtils.validateResponse(conn, HttpURLConnection.HTTP_OK); + } + + /** * Renames Path src to Path dst. Can take place on local fs * or remote DFS. */ Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java?rev=1441606&r1=1441605&r2=1441606&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/FSOperations.java Fri Feb 1 19:46:14 2013 @@ -199,6 +199,47 @@ public class FSOperations { } /** + * Executor that performs an append FileSystemAccess files system operation. + */ + @InterfaceAudience.Private + public static class FSConcat implements FileSystemAccess.FileSystemExecutor<Void> { + private Path path; + private Path[] sources; + + /** + * Creates a Concat executor. + * + * @param path target path to concat to. + * @param sources comma seperated absolute paths to use as sources. + */ + public FSConcat(String path, String[] sources) { + this.sources = new Path[sources.length]; + + for(int i = 0; i < sources.length; i++) { + this.sources[i] = new Path(sources[i]); + } + + this.path = new Path(path); + } + + /** + * Executes the filesystem operation. + * + * @param fs filesystem instance to use. + * + * @return void. + * + * @throws IOException thrown if an IO error occured. + */ + @Override + public Void execute(FileSystem fs) throws IOException { + fs.concat(path, sources); + return null; + } + + } + + /** * Executor that performs a content-summary FileSystemAccess files system operation. */ @InterfaceAudience.Private Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java?rev=1441606&r1=1441605&r2=1441606&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSParametersProvider.java Fri Feb 1 19:46:14 2013 @@ -58,6 +58,7 @@ public class HttpFSParametersProvider ex PARAMS_DEF.put(Operation.INSTRUMENTATION, new Class[]{DoAsParam.class}); PARAMS_DEF.put(Operation.APPEND, new Class[]{DoAsParam.class, DataParam.class}); + PARAMS_DEF.put(Operation.CONCAT, new Class[]{SourcesParam.class}); PARAMS_DEF.put(Operation.CREATE, new Class[]{DoAsParam.class, PermissionParam.class, OverwriteParam.class, ReplicationParam.class, BlockSizeParam.class, DataParam.class}); @@ -389,6 +390,25 @@ public class HttpFSParametersProvider ex } /** + * Class for concat sources parameter. + */ + @InterfaceAudience.Private + public static class SourcesParam extends StringParam { + + /** + * Parameter name. + */ + public static final String NAME = HttpFSFileSystem.SOURCES_PARAM; + + /** + * Constructor. + */ + public SourcesParam() { + super(NAME, null); + } + } + + /** * Class for to-path parameter. */ @InterfaceAudience.Private Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java?rev=1441606&r1=1441605&r2=1441606&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/java/org/apache/hadoop/fs/http/server/HttpFSServer.java Fri Feb 1 19:46:14 2013 @@ -22,22 +22,23 @@ import org.apache.hadoop.classification. import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.http.client.HttpFSFileSystem; -import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OperationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.AccessTimeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.BlockSizeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DataParam; -import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.RecursiveParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DestinationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DoAsParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.FilterParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.GroupParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.LenParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ModifiedTimeParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OffsetParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OperationParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OverwriteParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.OwnerParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.PermissionParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.RecursiveParam; import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.ReplicationParam; -import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.DestinationParam; +import org.apache.hadoop.fs.http.server.HttpFSParametersProvider.SourcesParam; import org.apache.hadoop.lib.service.FileSystemAccess; import org.apache.hadoop.lib.service.FileSystemAccessException; import org.apache.hadoop.lib.service.Groups; @@ -403,9 +404,9 @@ public class HttpFSServer { Response response; path = makeAbsolute(path); MDC.put(HttpFSFileSystem.OP_PARAM, op.value().name()); - String doAs = params.get(DoAsParam.NAME, DoAsParam.class); switch (op.value()) { case APPEND: { + String doAs = params.get(DoAsParam.NAME, DoAsParam.class); Boolean hasData = params.get(DataParam.NAME, DataParam.class); if (!hasData) { response = Response.temporaryRedirect( @@ -420,6 +421,18 @@ public class HttpFSServer { } break; } + case CONCAT: { + System.out.println("HTTPFS SERVER CONCAT"); + String sources = params.get(SourcesParam.NAME, SourcesParam.class); + + FSOperations.FSConcat command = + new FSOperations.FSConcat(path, sources.split(",")); + fsExecute(user, null, command); + AUDIT_LOG.info("[{}]", path); + System.out.println("SENT RESPONSE"); + response = Response.ok().build(); + break; + } default: { throw new IOException( MessageFormat.format("Invalid HTTP POST operation [{0}]", Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java?rev=1441606&r1=1441605&r2=1441606&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java Fri Feb 1 19:46:14 2013 @@ -28,6 +28,8 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.http.server.HttpFSServerWebApp; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSConfigKeys; +import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.test.HFSTestCase; import org.apache.hadoop.test.HadoopUsersConfTestHelper; @@ -206,6 +208,30 @@ public abstract class BaseTestHttpFSWith } } + private void testConcat() throws Exception { + Configuration config = getProxiedFSConf(); + config.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024); + if (!isLocalFS()) { + FileSystem fs = FileSystem.get(config); + fs.mkdirs(getProxiedFSTestDir()); + Path path1 = new Path("/test/foo.txt"); + Path path2 = new Path("/test/bar.txt"); + Path path3 = new Path("/test/derp.txt"); + DFSTestUtil.createFile(fs, path1, 1024, (short) 3, 0); + DFSTestUtil.createFile(fs, path2, 1024, (short) 3, 0); + DFSTestUtil.createFile(fs, path3, 1024, (short) 3, 0); + fs.close(); + fs = getHttpFSFileSystem(); + fs.concat(path1, new Path[]{path2, path3}); + fs.close(); + fs = FileSystem.get(config); + Assert.assertTrue(fs.exists(path1)); + Assert.assertFalse(fs.exists(path2)); + Assert.assertFalse(fs.exists(path3)); + fs.close(); + } + } + private void testRename() throws Exception { FileSystem fs = FileSystem.get(getProxiedFSConf()); Path path = new Path(getProxiedFSTestDir(), "foo"); @@ -450,7 +476,7 @@ public abstract class BaseTestHttpFSWith } protected enum Operation { - GET, OPEN, CREATE, APPEND, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS, + GET, OPEN, CREATE, APPEND, CONCAT, RENAME, DELETE, LIST_STATUS, WORKING_DIRECTORY, MKDIRS, SET_TIMES, SET_PERMISSION, SET_OWNER, SET_REPLICATION, CHECKSUM, CONTENT_SUMMARY } @@ -468,6 +494,8 @@ public abstract class BaseTestHttpFSWith case APPEND: testAppend(); break; + case CONCAT: + testConcat(); case RENAME: testRename(); break; Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1441606&r1=1441605&r2=1441606&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Feb 1 19:46:14 2013 @@ -216,6 +216,8 @@ Release 2.0.3-alpha - Unreleased HDFS-3598. WebHDFS support for file concat. (Plamen Jeliazkov via shv) + HDFS-4456. Add concat to HttpFS and WebHDFS REST API docs. (plamenj2003 via tucu) + OPTIMIZATIONS HDFS-3429. DataNode reads checksums even if client does not need them (todd) Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java?rev=1441606&r1=1441605&r2=1441606&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/resources/ConcatSourcesParam.java Fri Feb 1 19:46:14 2013 @@ -21,7 +21,7 @@ package org.apache.hadoop.hdfs.web.resou /** The concat source paths parameter. */ public class ConcatSourcesParam extends StringParam { /** Parameter name. */ - public static final String NAME = "srcs"; + public static final String NAME = "sources"; public static final String DEFAULT = NULL; Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm?rev=1441606&r1=1441605&r2=1441606&view=diff ============================================================================== --- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm (original) +++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm Fri Feb 1 19:46:14 2013 @@ -109,6 +109,9 @@ WebHDFS REST API * {{{Append to a File}<<<APPEND>>>}} (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.append) + * {{{Concat File(s)}<<<CONCAT>>>}} + (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat) + * HTTP DELETE * {{{Delete a File/Directory}<<<DELETE>>>}} @@ -299,6 +302,32 @@ Content-Length: 0 {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.append +** {Concat File(s)} + + * Submit a HTTP POST request. + ++--------------------------------- +curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=CONCAT&sources=<SOURCES>" ++--------------------------------- + + The client receives a response with zero content length: + ++--------------------------------- +HTTP/1.1 200 OK +Content-Length: 0 ++--------------------------------- + + [] + + This REST API call is available as of Hadoop version 2.0.3. + Please note that <SOURCES> is a comma seperated list of absolute paths. + (Example: sources=/test/file1,/test/file2,/test/file3) + + See also: + {{{Sources}<<<sources>>>}}, + {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat + + ** {Open and Read a File} * Submit a HTTP GET request with automatically following redirects. @@ -1727,6 +1756,29 @@ var tokenProperties = {{{Set Replication Factor}<<<SETREPLICATION>>>}} +** {Sources} + +*----------------+-------------------------------------------------------------------+ +|| Name | <<<sources>>> | +*----------------+-------------------------------------------------------------------+ +|| Description | The comma seperated absolute paths used for concatenation. | +*----------------+-------------------------------------------------------------------+ +|| Type | String | +*----------------+-------------------------------------------------------------------+ +|| Default Value | \<empty\> | +*----------------+-------------------------------------------------------------------+ +|| Valid Values | A list of comma seperated absolute FileSystem paths without scheme and authority. | +*----------------+-------------------------------------------------------------------+ +|| Syntax | See the note in {{Delegation}}. | +*----------------+-------------------------------------------------------------------+ + + <<Note>> that sources are absolute FileSystem paths. + + + See also: + {{{Concat File(s)}<<<CONCAT>>>}} + + ** {Token} *----------------+-------------------------------------------------------------------+