Repository: metron Updated Branches: refs/heads/master 545f358a1 -> 95db8b40e
METRON-1176 REST: HDFS Service should support setting permissions on files when writing (ottobackwards) closes apache/metron#749 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/95db8b40 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/95db8b40 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/95db8b40 Branch: refs/heads/master Commit: 95db8b40e76d07aa4344d7b05f0f0c14ed24f59f Parents: 545f358 Author: ottobackwards <ottobackwa...@gmail.com> Authored: Tue Sep 12 11:04:07 2017 -0400 Committer: otto <o...@apache.org> Committed: Tue Sep 12 11:04:07 2017 -0400 ---------------------------------------------------------------------- metron-interface/metron-rest/README.md | 5 ++- .../metron/rest/controller/HdfsController.java | 11 ++++-- .../apache/metron/rest/service/HdfsService.java | 2 +- .../rest/service/impl/GrokServiceImpl.java | 2 +- .../rest/service/impl/HdfsServiceImpl.java | 39 +++++++++++++++----- .../impl/HdfsServiceImplExceptionTest.java | 12 +++++- .../rest/service/impl/HdfsServiceImplTest.java | 16 +++++++- 7 files changed, 70 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index ba75ee1..27e04a3 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -296,10 +296,13 @@ Request and Response objects are JSON formatted. The JSON schemas are available * 200 - JSON results ### `POST /api/v1/hdfs` - * Description: Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists. + * Description: Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists. Permissions must be set for all three groups if they are to be set. If any are missing, the default permissions will be used, and if any are invalid an exception will be thrown. * Input: * path - Path to HDFS file * contents - File contents + * userMode - [optional] symbolic permission string for user portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute + * groupMode - [optional] symbolic permission string for group portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute + * otherMode - [optional] symbolic permission string for other portion of the permissions to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is used to exclude that permission such as 'rw-' for read, write, no execute * Returns: * 200 - Contents were written http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java index b8957a9..80b5942 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java @@ -67,9 +67,14 @@ public class HdfsController { @ApiOperation(value = "Writes contents to an HDFS file. Warning: this will overwrite the contents of a file if it already exists.") @ApiResponse(message = "Contents were written", code = 200) @RequestMapping(method = RequestMethod.POST) - ResponseEntity<Void> write(@ApiParam(name="path", value="Path to HDFS file", required=true) @RequestParam String path, - @ApiParam(name="contents", value="File contents", required=true) @RequestBody String contents) throws RestException { - hdfsService.write(new Path(path), contents.getBytes(UTF_8)); + ResponseEntity<Void> write( + @ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam String path, + @ApiParam(name = "contents", value = "File contents", required = true) @RequestBody String contents, + @ApiParam(name = "userMode", value = "requested user permissions") @RequestParam(required = false, defaultValue = "") String userMode, + @ApiParam(name = "groupMode", value = "requested group permissions") @RequestParam(required = false, defaultValue = "") String groupMode, + @ApiParam(name = "otherMode", value = "requested other permissions") @RequestParam(required = false, defaultValue = "") String otherMode + ) throws RestException { + hdfsService.write(new Path(path), contents.getBytes(UTF_8), userMode, groupMode, otherMode); return new ResponseEntity<>(HttpStatus.OK); } http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java index 58dbf9b..2748e66 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java @@ -26,7 +26,7 @@ public interface HdfsService { String read(Path path) throws RestException; - void write(Path path, byte[] contents) throws RestException; + void write(Path path, byte[] contents,String userMode, String groupMode, String otherMode) throws RestException; List<String> list(Path path) throws RestException; http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java index e185b53..4a39635 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java @@ -91,7 +91,7 @@ public class GrokServiceImpl implements GrokService { if (statement != null) { Path path = getTemporaryGrokRootPath(); hdfsService.mkdirs(path); - hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8"))); + hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8")),null,null,null); return path; } else { throw new RestException("A grokStatement must be provided"); http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java index a9ae8eb..d637ecf 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java @@ -17,10 +17,13 @@ */ package org.apache.metron.rest.service.impl; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.IOUtils; import org.apache.metron.rest.RestException; import org.apache.metron.rest.service.HdfsService; @@ -68,17 +71,35 @@ public class HdfsServiceImpl implements HdfsService { return new String(byteArrayOutputStream.toByteArray(), UTF_8); } - @Override - public void write(Path path, byte[] contents) throws RestException { - FSDataOutputStream fsDataOutputStream; - try { - fsDataOutputStream = FileSystem.get(configuration).create(path, true); - fsDataOutputStream.write(contents); - fsDataOutputStream.close(); - } catch (IOException e) { - throw new RestException(e); + @Override + public void write(Path path, byte[] contents, String userMode, String groupMode, String otherMode) + throws RestException { + FSDataOutputStream fsDataOutputStream; + try { + FsPermission permission = null; + boolean setPermissions = false; + if(StringUtils.isNotEmpty(userMode) && StringUtils.isNotEmpty(groupMode) && StringUtils.isNotEmpty(otherMode)) { + // invalid actions will return null + FsAction userAction = FsAction.getFsAction(userMode); + FsAction groupAction = FsAction.getFsAction(groupMode); + FsAction otherAction = FsAction.getFsAction(otherMode); + if(userAction == null || groupAction == null || otherAction == null){ + throw new RestException(String.format("Invalid permission set: user[%s] " + + "group[%s] other[%s]", userAction, groupAction, otherAction)); + } + permission = new FsPermission(userAction, groupAction, otherAction); + setPermissions = true; + } + fsDataOutputStream = FileSystem.get(configuration).create(path, true); + fsDataOutputStream.write(contents); + fsDataOutputStream.close(); + if(setPermissions) { + FileSystem.get(configuration).setPermission(path, permission); } + } catch (IOException e) { + throw new RestException(e); } + } @Override public boolean delete(Path path, boolean recursive) throws RestException { http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java index d11f5a4..7282859 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java @@ -85,7 +85,17 @@ public class HdfsServiceImplExceptionTest { when(FileSystem.get(configuration)).thenReturn(fileSystem); when(fileSystem.create(new Path(testDir), true)).thenThrow(new IOException()); - hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8)); + hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8),null, null,null); + } + + @Test + public void writeShouldThrowIfInvalidPermissions() throws Exception { + exception.expect(RestException.class); + + FileSystem fileSystem = mock(FileSystem.class); + when(FileSystem.get(configuration)).thenReturn(fileSystem); + + hdfsService.write(new Path(testDir,"test"),"oops".getBytes(UTF_8), "foo", "r-x","r--"); } @Test http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java index 005a427..280be63 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java @@ -19,7 +19,10 @@ package org.apache.metron.rest.service.impl; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.fs.permission.FsAction; import org.apache.metron.rest.service.HdfsService; import org.junit.After; import org.junit.Before; @@ -84,12 +87,23 @@ public class HdfsServiceImplTest { @Test public void writeShouldProperlyWriteContents() throws Exception { String contents = "contents"; - hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8)); + hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8),null,null,null); assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest.txt"))); } @Test + public void writeShouldProperlyWriteContentsWithPermissions() throws Exception { + String contents = "contents"; + Path thePath = new Path(testDir,"writeTest2.txt"); + hdfsService.write(thePath, contents.getBytes(UTF_8),"rw-","r-x","r--"); + + assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest2.txt"))); + assertEquals(FileSystem.get(configuration).listStatus(thePath)[0].getPermission().toShort(), + new FsPermission(FsAction.READ_WRITE,FsAction.READ_EXECUTE,FsAction.READ).toShort()); + } + + @Test public void deleteShouldProperlyDeleteFile() throws Exception { String contents = "contents"; FileUtils.writeStringToFile(new File(testDir, "deleteTest.txt"), contents);
