metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject metron git commit: METRON-1176 REST: HDFS Service should support setting permissions on files when writing (ottobackwards) closes apache/metron#749
Date Tue, 12 Sep 2017 15:33:41 GMT
Repository: metron
Updated Branches:
  refs/heads/master 545f358a1 -> 95db8b40e


METRON-1176 REST: HDFS Service should support setting permissions on files when writing (ottobackwards)
closes apache/metron#749


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/95db8b40
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/95db8b40
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/95db8b40

Branch: refs/heads/master
Commit: 95db8b40e76d07aa4344d7b05f0f0c14ed24f59f
Parents: 545f358
Author: ottobackwards <ottobackwards@gmail.com>
Authored: Tue Sep 12 11:04:07 2017 -0400
Committer: otto <otto@apache.org>
Committed: Tue Sep 12 11:04:07 2017 -0400

----------------------------------------------------------------------
 metron-interface/metron-rest/README.md          |  5 ++-
 .../metron/rest/controller/HdfsController.java  | 11 ++++--
 .../apache/metron/rest/service/HdfsService.java |  2 +-
 .../rest/service/impl/GrokServiceImpl.java      |  2 +-
 .../rest/service/impl/HdfsServiceImpl.java      | 39 +++++++++++++++-----
 .../impl/HdfsServiceImplExceptionTest.java      | 12 +++++-
 .../rest/service/impl/HdfsServiceImplTest.java  | 16 +++++++-
 7 files changed, 70 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index ba75ee1..27e04a3 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -296,10 +296,13 @@ Request and Response objects are JSON formatted.  The JSON schemas are
available
     * 200 - JSON results
 
 ### `POST /api/v1/hdfs`
-  * Description: Writes contents to an HDFS file.  Warning: this will overwrite the contents
of a file if it already exists.
+  * Description: Writes contents to an HDFS file.  Warning: this will overwrite the contents
of a file if it already exists. Permissions must be set for all three groups if they are to
be set. If any are missing, the default permissions will be used, and if any are invalid an
exception will be thrown.
   * Input:
     * path - Path to HDFS file
     * contents - File contents
+    * userMode - [optional] symbolic permission string for user portion of the permissions
to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is
used to exclude that permission such as 'rw-' for read, write, no execute
+    * groupMode - [optional] symbolic permission string for group portion of the permissions
to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is
used to exclude that permission such as 'rw-' for read, write, no execute
+    * otherMode - [optional] symbolic permission string for other portion of the permissions
to be set on the file written. For example 'rwx' or read, write, execute. The symbol '-' is
used to exclude that permission such as 'rw-' for read, write, no execute
   * Returns:
     * 200 - Contents were written
 

http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
index b8957a9..80b5942 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/HdfsController.java
@@ -67,9 +67,14 @@ public class HdfsController {
   @ApiOperation(value = "Writes contents to an HDFS file.  Warning: this will overwrite the
contents of a file if it already exists.")
   @ApiResponse(message = "Contents were written", code = 200)
   @RequestMapping(method = RequestMethod.POST)
-  ResponseEntity<Void> write(@ApiParam(name="path", value="Path to HDFS file", required=true)
@RequestParam String path,
-                                              @ApiParam(name="contents", value="File contents",
required=true) @RequestBody String contents) throws RestException {
-    hdfsService.write(new Path(path), contents.getBytes(UTF_8));
+  ResponseEntity<Void> write(
+      @ApiParam(name = "path", value = "Path to HDFS file", required = true) @RequestParam
String path,
+      @ApiParam(name = "contents", value = "File contents", required = true) @RequestBody
String contents,
+      @ApiParam(name = "userMode", value = "requested user permissions") @RequestParam(required
= false, defaultValue = "") String userMode,
+      @ApiParam(name = "groupMode", value = "requested group permissions") @RequestParam(required
= false, defaultValue = "") String groupMode,
+      @ApiParam(name = "otherMode", value = "requested other permissions") @RequestParam(required
= false, defaultValue = "") String otherMode
+      ) throws RestException {
+    hdfsService.write(new Path(path), contents.getBytes(UTF_8), userMode, groupMode, otherMode);
     return new ResponseEntity<>(HttpStatus.OK);
 
   }

http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
index 58dbf9b..2748e66 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/HdfsService.java
@@ -26,7 +26,7 @@ public interface HdfsService {
 
     String read(Path path) throws RestException;
 
-    void write(Path path, byte[] contents) throws RestException;
+    void write(Path path, byte[] contents,String userMode, String groupMode, String otherMode)
throws RestException;
 
     List<String> list(Path path) throws RestException;
 

http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
index e185b53..4a39635 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/GrokServiceImpl.java
@@ -91,7 +91,7 @@ public class GrokServiceImpl implements GrokService {
         if (statement != null) {
             Path path = getTemporaryGrokRootPath();
             hdfsService.mkdirs(path);
-            hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8")));
+            hdfsService.write(new Path(path, name), statement.getBytes(Charset.forName("utf-8")),null,null,null);
             return path;
         } else {
             throw new RestException("A grokStatement must be provided");

http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
index a9ae8eb..d637ecf 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/HdfsServiceImpl.java
@@ -17,10 +17,13 @@
  */
 package org.apache.metron.rest.service.impl;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.metron.rest.RestException;
 import org.apache.metron.rest.service.HdfsService;
@@ -68,17 +71,35 @@ public class HdfsServiceImpl implements HdfsService {
       return new String(byteArrayOutputStream.toByteArray(), UTF_8);
     }
 
-    @Override
-    public void write(Path path, byte[] contents) throws RestException {
-      FSDataOutputStream fsDataOutputStream;
-      try {
-        fsDataOutputStream = FileSystem.get(configuration).create(path, true);
-        fsDataOutputStream.write(contents);
-        fsDataOutputStream.close();
-      } catch (IOException e) {
-        throw new RestException(e);
+  @Override
+  public void write(Path path, byte[] contents, String userMode, String groupMode, String
otherMode)
+      throws RestException {
+    FSDataOutputStream fsDataOutputStream;
+    try {
+      FsPermission permission = null;
+      boolean setPermissions = false;
+      if(StringUtils.isNotEmpty(userMode) && StringUtils.isNotEmpty(groupMode) &&
StringUtils.isNotEmpty(otherMode)) {
+        // invalid actions will return null
+        FsAction userAction = FsAction.getFsAction(userMode);
+        FsAction groupAction = FsAction.getFsAction(groupMode);
+        FsAction otherAction = FsAction.getFsAction(otherMode);
+        if(userAction == null || groupAction == null || otherAction == null){
+          throw new RestException(String.format("Invalid permission set: user[%s] " +
+              "group[%s] other[%s]", userAction, groupAction, otherAction));
+        }
+        permission = new FsPermission(userAction, groupAction, otherAction);
+        setPermissions = true;
+      }
+      fsDataOutputStream = FileSystem.get(configuration).create(path, true);
+      fsDataOutputStream.write(contents);
+      fsDataOutputStream.close();
+      if(setPermissions) {
+        FileSystem.get(configuration).setPermission(path, permission);
       }
+    } catch (IOException e) {
+      throw new RestException(e);
     }
+  }
 
     @Override
     public boolean delete(Path path, boolean recursive) throws RestException {

http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
index d11f5a4..7282859 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplExceptionTest.java
@@ -85,7 +85,17 @@ public class HdfsServiceImplExceptionTest {
         when(FileSystem.get(configuration)).thenReturn(fileSystem);
         when(fileSystem.create(new Path(testDir), true)).thenThrow(new IOException());
 
-        hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8));
+        hdfsService.write(new Path(testDir), "contents".getBytes(UTF_8),null, null,null);
+    }
+
+    @Test
+    public void writeShouldThrowIfInvalidPermissions() throws Exception {
+        exception.expect(RestException.class);
+
+        FileSystem fileSystem = mock(FileSystem.class);
+        when(FileSystem.get(configuration)).thenReturn(fileSystem);
+
+        hdfsService.write(new Path(testDir,"test"),"oops".getBytes(UTF_8), "foo", "r-x","r--");
     }
 
     @Test

http://git-wip-us.apache.org/repos/asf/metron/blob/95db8b40/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
index 005a427..280be63 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/HdfsServiceImplTest.java
@@ -19,7 +19,10 @@ package org.apache.metron.rest.service.impl;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.metron.rest.service.HdfsService;
 import org.junit.After;
 import org.junit.Before;
@@ -84,12 +87,23 @@ public class HdfsServiceImplTest {
     @Test
     public void writeShouldProperlyWriteContents() throws Exception {
         String contents = "contents";
-        hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8));
+        hdfsService.write(new Path(testDir, "writeTest.txt"), contents.getBytes(UTF_8),null,null,null);
 
         assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest.txt")));
     }
 
     @Test
+    public void writeShouldProperlyWriteContentsWithPermissions() throws Exception {
+        String contents = "contents";
+        Path thePath = new Path(testDir,"writeTest2.txt");
+        hdfsService.write(thePath, contents.getBytes(UTF_8),"rw-","r-x","r--");
+
+        assertEquals("contents", FileUtils.readFileToString(new File(testDir, "writeTest2.txt")));
+        assertEquals(FileSystem.get(configuration).listStatus(thePath)[0].getPermission().toShort(),
+        new FsPermission(FsAction.READ_WRITE,FsAction.READ_EXECUTE,FsAction.READ).toShort());
+    }
+
+    @Test
     public void deleteShouldProperlyDeleteFile() throws Exception {
         String contents = "contents";
         FileUtils.writeStringToFile(new File(testDir, "deleteTest.txt"), contents);


Mime
View raw message