Mailing-List: contact commits-help@eagle.incubator.apache.org; run by ezmlm
Precedence: bulk
Reply-To: dev@eagle.incubator.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit
From: qingwzhao@apache.org
To: commits@eagle.incubator.apache.org
Date: Wed, 19 Oct 2016 03:19:09 -0000
Message-Id: <51b92bcb6ae64b949290ad7c596e5067@git.apache.org>
In-Reply-To: <6d3efa645f734184b21f4bdb62298628@git.apache.org>
References: <6d3efa645f734184b21f4bdb62298628@git.apache.org>
Subject: [10/28] incubator-eagle git commit: EAGLE-307 Update
 "logstash-kafka-conf.md" Update "logstash-kafka-conf.md"
archived-at: Wed, 19 Oct 2016 03:19:07 -0000

EAGLE-307 Update "logstash-kafka-conf.md"
Update "logstash-kafka-conf.md"

Author: @DadanielZ <DadanielZ>
Reviewer: @yonzhang <yonzhang2012@gmail.com>

Closes: #234


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4627eb0b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4627eb0b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4627eb0b

Branch: refs/heads/master
Commit: 4627eb0b576dec25c8c97d57410a044dcbd6af05
Parents: 36422ea
Author: yonzhang <yonzhang2012@gmail.com>
Authored: Thu Jun 16 15:06:36 2016 -0700
Committer: yonzhang <yonzhang2012@gmail.com>
Committed: Thu Jun 16 15:06:36 2016 -0700

----------------------------------------------------------------------
 .../src/main/docs/logstash-kafka-conf.md        | 19 ++++-
 .../src/main/resources/maprFSAuditLog-init.sh   |  2 +-
 .../security/hdfs/MAPRFSResourceConstants.java  | 24 +++++++
 .../hdfs/rest/MAPRFSResourceWebResource.java    | 73 ++++++++++++++++++++
 4 files changed, 115 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4627eb0b/eagle-assembly/src/main/docs/logstash-kafka-conf.md
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/docs/logstash-kafka-conf.md b/eagle-assembly/src/main/docs/logstash-kafka-conf.md
index 9003fb4..342302b 100644
--- a/eagle-assembly/src/main/docs/logstash-kafka-conf.md
+++ b/eagle-assembly/src/main/docs/logstash-kafka-conf.md
@@ -120,7 +120,7 @@ The 2.0 release of Logstash includes a new version of the Kafka output plugin wi
                 type => "hdp-nn-audit"
                 path => "/path/to/audit.log"
                 start_position => end
-                sincedb_path => "/var/log/logstash/"
+                sincedb_path => "/opt/logstash/sincedb.txt"
              }
         }
 
@@ -163,7 +163,7 @@ The 2.0 release of Logstash includes a new version of the Kafka output plugin wi
 				type => "hdp-nn-audit"
 				path => "/path/to/audit.log"
 				start_position => end
-				sincedb_path => "/var/log/logstash/"
+				sincedb_path => "/opt/logstash/sincedb.txt"
 			}
 		}
 
@@ -196,6 +196,21 @@ The 2.0 release of Logstash includes a new version of the Kafka output plugin wi
 			  }
 		}
 
+
+Notice:
+ `path => "/path/to/audit.log"` : 
+ Here `path` can be configured to be a single string or an array. For example:
+    `path => [ "var/msg/example.out", /var/message/*.out", "/var/log/*.log"]`
+    `path => "/var/log/hadoop/hdfs/hdfs-audit.log.*"`
+As you can see, we can apply regex pattern to match mutiple files.
+
+
+`sincedb_path => "/opt/logstash/sincedb.txt"`: 
+`sincedb_path` must be a file writable by logstash. This file will be used by Logstash to keep track of the current position in each file.
+
+
+
+
 #### grok pattern testing
 We have 3 typical patterns for ugi field as follows
 2015-02-11 15:00:00,000 INFO FSNamesystem.audit: allowed=true	ugi=user1@xyz.com (auth:TOKEN)	ip=/10.115.44.55	cmd=open	src=/apps/hdmi-technology/b_pulsar_coe/schema/avroschema/Session.avsc	dst=null	perm=null

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4627eb0b/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh b/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh
index 5f99190..19eed8d 100644
--- a/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh
+++ b/eagle-security/eagle-security-maprfs-auditlog/src/main/resources/maprFSAuditLog-init.sh
@@ -48,7 +48,7 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a
         "alias":"MapRFSAuditLogMonitor",
         "groupName":"MapR",
         "features":["common","metadata", "classification"],
-	"config":"{\n\t\"view\": {\n\t\t\"prefix\": \"fileSensitivity\",\n\t\t\"service\": \"FileSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"filedir\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"folder\",\n\t\t\"api\": \"hdfsResource\"\n\t}\n}"
+	"config":"{\n\t\"view\": {\n\t\t\"prefix\": \"fileSensitivity\",\n\t\t\"service\": \"FileSensitivityService\",\n\t\t\"keys\": [\n\t\t\t\"filedir\",\n\t\t\t\"sensitivityType\"\n\t\t],\n\t\t\"type\": \"folder\",\n\t\t\"api\": \"maprfsResource\"\n\t}\n}"
      }
   ]
   '

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4627eb0b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/MAPRFSResourceConstants.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/MAPRFSResourceConstants.java b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/MAPRFSResourceConstants.java
new file mode 100644
index 0000000..726ea8b
--- /dev/null
+++ b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/MAPRFSResourceConstants.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.hdfs;
+
+public class MAPRFSResourceConstants {
+
+    public static final String MAPRFS_RESOURCE="/maprfsResource";
+    public static final String MAPRFS_APPLICATION="MAPRFSAuditLog";
+    public static final String MAPRFS_RESOURCE_RESOLVE_FORMAT_HINT ="MAPRFS Resource must be start with /";
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4627eb0b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/MAPRFSResourceWebResource.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/MAPRFSResourceWebResource.java b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/MAPRFSResourceWebResource.java
new file mode 100644
index 0000000..11206bb
--- /dev/null
+++ b/eagle-security/eagle-security-maprfs-web/src/main/java/org/apache/eagle/service/security/hdfs/rest/MAPRFSResourceWebResource.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.eagle.service.security.hdfs.rest;
+
+
+import com.typesafe.config.Config;
+import org.apache.eagle.security.entity.FileStatusEntity;
+import org.apache.eagle.security.resolver.MetadataAccessConfigRepo;
+import org.apache.eagle.service.common.EagleExceptionWrapper;
+import org.apache.eagle.service.security.hdfs.HDFSFileSystem;
+import org.apache.eagle.service.security.hdfs.HDFSResourceSensitivityDataJoiner;
+import org.apache.eagle.service.security.hdfs.MAPRFSResourceConstants;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.ws.rs.*;
+import javax.ws.rs.core.MediaType;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * REST Web Service to browse files and Paths in MAPRFS
+ */
+@Path(MAPRFSResourceConstants.MAPRFS_RESOURCE)
+public class MAPRFSResourceWebResource
+{
+    private static Logger LOG = LoggerFactory.getLogger(MAPRFSResourceWebResource.class);
+
+    @GET
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Produces(MediaType.APPLICATION_JSON)
+    public HDFSResourceWebResponse getHDFSResource(@QueryParam("site") String site , @QueryParam("path") String filePath )
+    {
+        LOG.info("Starting MAPRFS Resource Browsing.  Query Parameters ==> Site :"+site+"  Path : "+filePath );
+        HDFSResourceWebResponse response = new HDFSResourceWebResponse();
+        HDFSResourceWebRequestValidator validator = new HDFSResourceWebRequestValidator();
+        MetadataAccessConfigRepo repo = new MetadataAccessConfigRepo();
+        List<FileStatusEntity> result = new ArrayList<>();
+        List<FileStatus> fileStatuses = null;
+        try {
+            validator.validate(site, filePath); // First Step would be validating Request
+            Config config = repo.getConfig(MAPRFSResourceConstants.MAPRFS_APPLICATION, site);
+            Configuration conf = repo.convert(config);
+            HDFSFileSystem fileSystem = new HDFSFileSystem(conf);
+            fileStatuses = fileSystem.browse(filePath);
+            // Join with File Sensitivity Info
+            HDFSResourceSensitivityDataJoiner joiner = new HDFSResourceSensitivityDataJoiner();
+            result = joiner.joinFileSensitivity(site, fileStatuses);
+            LOG.info("Successfully browsed files in MAPRFS .");
+        } catch( Exception ex ) {
+            response.setException(EagleExceptionWrapper.wrap(ex));
+            LOG.error(" Exception When browsing Files for the MAPRFS Path  :"+filePath+"  " , ex);
+        }
+        response.setObj(result);
+        return response;
+    }
+}