eagle-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From h..@apache.org
Subject [1/5] incubator-eagle git commit: change package name
Date Mon, 30 Nov 2015 02:48:58 GMT
Repository: incubator-eagle
Updated Branches:
  refs/heads/master 5bffdfd3a -> dfcf14098


change package name


Project: http://git-wip-us.apache.org/repos/asf/incubator-eagle/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-eagle/commit/4329ef12
Tree: http://git-wip-us.apache.org/repos/asf/incubator-eagle/tree/4329ef12
Diff: http://git-wip-us.apache.org/repos/asf/incubator-eagle/diff/4329ef12

Branch: refs/heads/master
Commit: 4329ef12365fbf163fc91126271ddad0d54198b9
Parents: 5bffdfd
Author: Zhao, Qingwen <qingwzhao@ebay.com>
Authored: Wed Nov 25 19:38:30 2015 +0800
Committer: Zhao, Qingwen <qingwzhao@ebay.com>
Committed: Wed Nov 25 19:41:32 2015 +0800

----------------------------------------------------------------------
 eagle-assembly/src/main/bin/eagle-topology.sh   |  3 +-
 .../bin/hdfs-securitylog-metadata-create.sh     |  2 +-
 .../sandbox-hdfsSecurityLog-application.conf    |  2 +-
 .../HDFSSecurityLogKafkaDeserializer.java       | 63 ---------------
 .../HDFSSecurityLogProcessorMain.java           | 50 ------------
 .../parse/HDFSSecurityLogObject.java            | 27 -------
 .../parse/HDFSSecurityLogParser.java            | 82 --------------------
 .../HDFSSecurityLogKafkaDeserializer.java       | 63 +++++++++++++++
 .../HDFSSecurityLogProcessorMain.java           | 50 ++++++++++++
 .../parse/HDFSSecurityLogObject.java            | 27 +++++++
 .../parse/HDFSSecurityLogParser.java            | 82 ++++++++++++++++++++
 .../src/main/resources/application.conf         |  2 +-
 .../security/TestHDFSSecuritylogParser.java     | 41 ----------
 .../security/TestHDFSSecuritylogParser.java     | 41 ++++++++++
 14 files changed, 268 insertions(+), 267 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-assembly/src/main/bin/eagle-topology.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/eagle-topology.sh b/eagle-assembly/src/main/bin/eagle-topology.sh
index c8625e0..ebcef9c 100755
--- a/eagle-assembly/src/main/bin/eagle-topology.sh
+++ b/eagle-assembly/src/main/bin/eagle-topology.sh
@@ -111,7 +111,8 @@ done
 
 
 if [ -z "$jarName" ]; then
-     jarName="${EAGLE_HOME}/lib/topology/eagle-topology-0.1.0-assembly.jar"
+    echo "Error: jar file is not found"
+    exit 1
 fi
 
 if [ -z "$mainClass" ]; then

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh b/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh
index 113bc66..7aa09a9 100644
--- a/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh
+++ b/eagle-assembly/src/main/bin/hdfs-securitylog-metadata-create.sh
@@ -31,7 +31,7 @@ curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:a
 
 #### AlertStreamSchemaService: schema for event from alert stream
 
-curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService"
-d '[{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"timestamp"},"attrDescription":"milliseconds
of the datetime","attrType":"long","category":"","attrValueResolver":""},{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"user"},"attrDescription":"","attrType":"string","category":"","attrValueResolver":"eagle.service.security.hbase.resolver.HbaseRequestResolver"},{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"allowed"},"attrDescription":"true,
false or none","attrType":"bool","category":"","attrValueResolver":""}]'
+curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertStreamSchemaService"
-d '[{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"timestamp"},"attrDescription":"milliseconds
of the datetime","attrType":"long","category":"","attrValueResolver":""},{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"user"},"attrDescription":"","attrType":"string","category":"","attrValueResolver":""},{"prefix":"alertStreamSchema","tags":{"dataSource":"hdfsSecurityLog","streamName":"hdfsSecurityLogEventStream","attrName":"allowed"},"attrDescription":"true,
false or none","attrType":"bool","category":"","attrValueResolver":""}]'
 
 curl -u ${EAGLE_SERVICE_USER}:${EAGLE_SERVICE_PASSWD} -X POST -H 'Content-Type:application/json'
"http://${EAGLE_SERVICE_HOST}:${EAGLE_SERVICE_PORT}/eagle-service/rest/entities?serviceName=AlertDataSourceService"
-d '[{"prefix":"alertDataSource","tags":{"site":"sandbox","dataSource":"hdfsSecurityLog"},"enabled":"true","desc":"HDFS
Security"}]'
 

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf
----------------------------------------------------------------------
diff --git a/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf b/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf
index bc2baf8..c70d811 100644
--- a/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf
+++ b/eagle-assembly/src/main/conf/sandbox-hdfsSecurityLog-application.conf
@@ -30,7 +30,7 @@
     "brokerZkPath" : "/brokers",
     "zkConnectionTimeoutMS" : 15000,
     "fetchSize" : 1048586,
-    "deserializerClass" : "eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer",
+    "deserializerClass" : "org.apache.eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer",
     "transactionZKServers" : "127.0.0.1",
     "transactionZKPort" : 2181,
     "transactionZKRoot" : "/consumers",

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
deleted file mode 100644
index 2c54f9a..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security.securitylog;
-
-
-import eagle.security.securitylog.parse.HDFSSecurityLogObject;
-import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer;
-import eagle.security.securitylog.parse.HDFSSecurityLogParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.Properties;
-import java.util.TreeMap;
-
-public class HDFSSecurityLogKafkaDeserializer implements SpoutKafkaMessageDeserializer {
-    Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogKafkaDeserializer.class);
-
-    private Properties props;
-
-    public  HDFSSecurityLogKafkaDeserializer(Properties props){
-        this.props = props;
-    }
-
-    @Override
-    public Object deserialize(byte[] arg0) {
-        String logLine = new String(arg0);
-
-        HDFSSecurityLogParser parser = new HDFSSecurityLogParser();
-        HDFSSecurityLogObject entity = null;
-        try{
-            entity = parser.parse(logLine);
-        }catch(Exception ex){
-            LOG.error("Failing parse audit log message", ex);
-        }
-        if(entity == null){
-            LOG.warn("Event ignored as it can't be correctly parsed, the log is ", logLine);
-            return null;
-        }
-        Map<String, Object> map = new TreeMap<String, Object>();
-        map.put("timestamp", entity.timestamp);
-        map.put("allowed", entity.allowed);
-        map.put("user", entity.user);
-
-        return map;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
deleted file mode 100644
index c87e1bc..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security.securitylog;
-
-
-import com.typesafe.config.Config;
-import com.typesafe.config.ConfigFactory;
-import com.typesafe.config.ConfigRenderOptions;
-import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
-import org.apache.eagle.dataproc.util.ConfigOptionParser;
-import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
-import org.apache.eagle.datastream.StormExecutionEnvironment;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class HDFSSecurityLogProcessorMain {
-    private static final Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogProcessorMain.class);
-
-    public static void main(String[] args) throws Exception{
-        new ConfigOptionParser().load(args);
-        System.setProperty("config.trace", "loads");
-        Config config = ConfigFactory.load();
-
-        LOG.info("Config class: " + config.getClass().getCanonicalName());
-
-        if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
-
-        StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
-
-        env.newSource(new KafkaSourcedSpoutProvider().getSpout(config)).renameOutputFields(1).withName("kafkaMsgConsumer")
-                .alertWithConsumer("hdfsSecurityLogEventStream", "hdfsSecurityLogAlertExecutor");
-        env.execute();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
deleted file mode 100644
index 5e3e8ba..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security.securitylog.parse;
-
-
-public class HDFSSecurityLogObject {
-    public long timestamp;
-    public Boolean allowed;
-    public String user;
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
deleted file mode 100644
index 144fb94..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security.securitylog.parse;
-
-
-import org.apache.eagle.security.util.LogParseUtil;
-import org.apache.eagle.common.DateTimeUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Serializable;
-import java.text.ParseException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class HDFSSecurityLogParser implements Serializable {
-    private final static Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogParser.class);
-
-    private final static int LOGDATE_INDEX = 1;
-    private final static int LOGATTRS_INDEX = 4;
-    private final static String SUCCESSFUL = "successful";
-    private final static String USER = "user";
-    private final static String ALLOWED = "allowed";
-    private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*):\\s+(.*)");
-
-    public HDFSSecurityLogObject parse(String logLine) throws ParseException {
-        Matcher logMatcher = loggerPattern.matcher(logLine);
-        if(!logMatcher.find()) return null;
-
-        HDFSSecurityLogObject ret = new HDFSSecurityLogObject();
-        String date = logMatcher.group(LOGDATE_INDEX);
-        ret.timestamp = DateTimeUtil.humanDateToMilliseconds(logMatcher.group(LOGDATE_INDEX));
-
-        Map<String, String> attrMap = parseAttr(logMatcher.group(LOGATTRS_INDEX));
-
-        if(attrMap == null) return null;
-
-        ret.user = attrMap.get(USER);
-        if(attrMap.get(ALLOWED).equals(ALLOWED)) {
-            ret.allowed = true;
-        } else {
-            ret.allowed = false;
-        }
-
-        return ret;
-    }
-
-    private Map<String, String> parseAttr(String logLine) {
-        if(logLine == null) return null;
-
-        Map<String, String> ret = new HashMap<>();
-        String [] fields = logLine.split("\\s+");
-
-        if(fields[1].equalsIgnoreCase(SUCCESSFUL)) {
-            ret.put(ALLOWED, "allowed");
-        } else {
-            ret.put(ALLOWED, "denied");
-        }
-
-        String user = LogParseUtil.parseUserFromUGI(fields[3]);
-        ret.put(USER, user);
-        return ret;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
new file mode 100644
index 0000000..5dada1e
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogKafkaDeserializer.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.securitylog;
+
+
+import org.apache.eagle.security.securitylog.parse.HDFSSecurityLogObject;
+import org.apache.eagle.dataproc.impl.storm.kafka.SpoutKafkaMessageDeserializer;
+import org.apache.eagle.security.securitylog.parse.HDFSSecurityLogParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+public class HDFSSecurityLogKafkaDeserializer implements SpoutKafkaMessageDeserializer {
+    Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogKafkaDeserializer.class);
+
+    private Properties props;
+
+    public  HDFSSecurityLogKafkaDeserializer(Properties props){
+        this.props = props;
+    }
+
+    @Override
+    public Object deserialize(byte[] arg0) {
+        String logLine = new String(arg0);
+
+        HDFSSecurityLogParser parser = new HDFSSecurityLogParser();
+        HDFSSecurityLogObject entity = null;
+        try{
+            entity = parser.parse(logLine);
+        }catch(Exception ex){
+            LOG.error("Failing parse audit log message", ex);
+        }
+        if(entity == null){
+            LOG.warn("Event ignored as it can't be correctly parsed, the log is ", logLine);
+            return null;
+        }
+        Map<String, Object> map = new TreeMap<String, Object>();
+        map.put("timestamp", entity.timestamp);
+        map.put("allowed", entity.allowed);
+        map.put("user", entity.user);
+
+        return map;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
new file mode 100644
index 0000000..b514a7c
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/HDFSSecurityLogProcessorMain.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.securitylog;
+
+
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigRenderOptions;
+import org.apache.eagle.dataproc.impl.storm.kafka.KafkaSourcedSpoutProvider;
+import org.apache.eagle.dataproc.util.ConfigOptionParser;
+import org.apache.eagle.datastream.ExecutionEnvironmentFactory;
+import org.apache.eagle.datastream.StormExecutionEnvironment;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HDFSSecurityLogProcessorMain {
+    private static final Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogProcessorMain.class);
+
+    public static void main(String[] args) throws Exception{
+        new ConfigOptionParser().load(args);
+        System.setProperty("config.trace", "loads");
+        Config config = ConfigFactory.load();
+
+        LOG.info("Config class: " + config.getClass().getCanonicalName());
+
+        if(LOG.isDebugEnabled()) LOG.debug("Config content:"+config.root().render(ConfigRenderOptions.concise()));
+
+        StormExecutionEnvironment env = ExecutionEnvironmentFactory.getStorm(config);
+
+        env.newSource(new KafkaSourcedSpoutProvider().getSpout(config)).renameOutputFields(1).withName("kafkaMsgConsumer")
+                .alertWithConsumer("hdfsSecurityLogEventStream", "hdfsSecurityLogAlertExecutor");
+        env.execute();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
new file mode 100644
index 0000000..fe19ff2
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogObject.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.securitylog.parse;
+
+
+public class HDFSSecurityLogObject {
+    public long timestamp;
+    public Boolean allowed;
+    public String user;
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
new file mode 100644
index 0000000..efd2371
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/java/org/apache/eagle/security/securitylog/parse/HDFSSecurityLogParser.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security.securitylog.parse;
+
+
+import org.apache.eagle.security.util.LogParseUtil;
+import org.apache.eagle.common.DateTimeUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.Serializable;
+import java.text.ParseException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class HDFSSecurityLogParser implements Serializable {
+    private final static Logger LOG = LoggerFactory.getLogger(HDFSSecurityLogParser.class);
+
+    private final static int LOGDATE_INDEX = 1;
+    private final static int LOGATTRS_INDEX = 4;
+    private final static String SUCCESSFUL = "successful";
+    private final static String USER = "user";
+    private final static String ALLOWED = "allowed";
+    private final static Pattern loggerPattern = Pattern.compile("^([\\d\\s\\-:,]+)\\s+(\\w+)\\s+(.*):\\s+(.*)");
+
+    public HDFSSecurityLogObject parse(String logLine) throws ParseException {
+        Matcher logMatcher = loggerPattern.matcher(logLine);
+        if(!logMatcher.find()) return null;
+
+        HDFSSecurityLogObject ret = new HDFSSecurityLogObject();
+        String date = logMatcher.group(LOGDATE_INDEX);
+        ret.timestamp = DateTimeUtil.humanDateToMilliseconds(logMatcher.group(LOGDATE_INDEX));
+
+        Map<String, String> attrMap = parseAttr(logMatcher.group(LOGATTRS_INDEX));
+
+        if(attrMap == null) return null;
+
+        ret.user = attrMap.get(USER);
+        if(attrMap.get(ALLOWED).equals(ALLOWED)) {
+            ret.allowed = true;
+        } else {
+            ret.allowed = false;
+        }
+
+        return ret;
+    }
+
+    private Map<String, String> parseAttr(String logLine) {
+        if(logLine == null) return null;
+
+        Map<String, String> ret = new HashMap<>();
+        String [] fields = logLine.split("\\s+");
+
+        if(fields[1].equalsIgnoreCase(SUCCESSFUL)) {
+            ret.put(ALLOWED, "allowed");
+        } else {
+            ret.put(ALLOWED, "denied");
+        }
+
+        String user = LogParseUtil.parseUserFromUGI(fields[3]);
+        ret.put(USER, user);
+        return ret;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
b/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
index fa970d1..9900ac8 100644
--- a/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/main/resources/application.conf
@@ -29,7 +29,7 @@
     "zkConnection" : "sandbox.hortonworks.com:2181",
     "zkConnectionTimeoutMS" : 15000,
     "fetchSize" : 1048586,
-    "deserializerClass" : "eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer",
+    "deserializerClass" : "org.apache.eagle.security.securitylog.HDFSSecurityLogKafkaDeserializer",
     "transactionZKServers" : "sandbox.hortonworks.com",
     "transactionZKPort" : 2181,
     "transactionZKRoot" : "/consumers",

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/test/java/eagle/security/TestHDFSSecuritylogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/test/java/eagle/security/TestHDFSSecuritylogParser.java
b/eagle-security/eagle-security-hdfs-securitylog/src/test/java/eagle/security/TestHDFSSecuritylogParser.java
deleted file mode 100644
index 4c79f86..0000000
--- a/eagle-security/eagle-security-hdfs-securitylog/src/test/java/eagle/security/TestHDFSSecuritylogParser.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package eagle.security;
-
-
-import eagle.security.securitylog.parse.HDFSSecurityLogObject;
-import eagle.security.securitylog.parse.HDFSSecurityLogParser;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.text.ParseException;
-
-public class TestHDFSSecuritylogParser {
-
-    @Test
-    public void test() throws ParseException {
-        String msg = "2015-11-18 08:41:10,200 INFO SecurityLogger.org.apache.hadoop.security.authorize.ServiceAuthorizationManager:
Authorization successful for hbase (auth:SIMPLE) for protocol=interface org.apache.hadoop.hdfs.protocol.ClientProtocol";
-
-        HDFSSecurityLogParser parser = new HDFSSecurityLogParser();
-        HDFSSecurityLogObject obj = parser.parse(msg);
-
-        Assert.assertEquals("hbase", obj.user);
-        Assert.assertEquals(true, obj.allowed);
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-eagle/blob/4329ef12/eagle-security/eagle-security-hdfs-securitylog/src/test/java/org/apache/eagle/security/TestHDFSSecuritylogParser.java
----------------------------------------------------------------------
diff --git a/eagle-security/eagle-security-hdfs-securitylog/src/test/java/org/apache/eagle/security/TestHDFSSecuritylogParser.java
b/eagle-security/eagle-security-hdfs-securitylog/src/test/java/org/apache/eagle/security/TestHDFSSecuritylogParser.java
new file mode 100644
index 0000000..b72d05e
--- /dev/null
+++ b/eagle-security/eagle-security-hdfs-securitylog/src/test/java/org/apache/eagle/security/TestHDFSSecuritylogParser.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.eagle.security;
+
+
+import org.apache.eagle.security.securitylog.parse.HDFSSecurityLogObject;
+import org.apache.eagle.security.securitylog.parse.HDFSSecurityLogParser;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.text.ParseException;
+
+public class TestHDFSSecuritylogParser {
+
+    @Test
+    public void test() throws ParseException {
+        String msg = "2015-11-18 08:41:10,200 INFO SecurityLogger.org.apache.hadoop.security.authorize.ServiceAuthorizationManager:
Authorization successful for hbase (auth:SIMPLE) for protocol=interface org.apache.hadoop.hdfs.protocol.ClientProtocol";
+
+        HDFSSecurityLogParser parser = new HDFSSecurityLogParser();
+        HDFSSecurityLogObject obj = parser.parse(msg);
+
+        Assert.assertEquals("hbase", obj.user);
+        Assert.assertEquals(true, obj.allowed);
+    }
+}


Mime
View raw message