ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [2/5] git commit: ARGUS-5: dev-test update for HDFS audit provider.
Date Tue, 23 Sep 2014 18:33:29 GMT
ARGUS-5: dev-test update for HDFS audit provider.

Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/ea4df438
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/ea4df438
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/ea4df438

Branch: refs/heads/master
Commit: ea4df438291cb981326056adb3eb3e1bf349c66f
Parents: 74d8393
Author: mneethiraj <mneethiraj@hortonworks.com>
Authored: Mon Sep 22 23:56:40 2014 -0700
Committer: mneethiraj <mneethiraj@hortonworks.com>
Committed: Mon Sep 22 23:56:40 2014 -0700

----------------------------------------------------------------------
 .../audit/provider/BufferedAuditProvider.java   |  8 ++++++
 .../audit/provider/LocalFileLogBuffer.java      | 14 ++++++++--
 .../com/xasecure/audit/provider/LogBuffer.java  |  2 ++
 .../audit/provider/hdfs/HdfsAuditProvider.java  | 17 +++++++++++-
 .../com/xasecure/audit/test/TestEvents.java     |  4 +++
 .../src/scripts/xasecure-audit.properties       | 29 ++++++++++++++++----
 6 files changed, 65 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
index 9b8cb40..3db5f91 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/BufferedAuditProvider.java
@@ -57,6 +57,14 @@ public abstract class BufferedAuditProvider implements AuditProvider {
 	public void flush() {
 	}
 
+	protected LogBuffer<AuditEventBase> getBuffer() {
+		return mBuffer;
+	}
+
+	protected LogDestination<AuditEventBase> getDestination() {
+		return mDestination;
+	}
+
 	protected void setBufferAndDestination(LogBuffer<AuditEventBase>      buffer,
 										   LogDestination<AuditEventBase> destination) {
 		mBuffer      = buffer;

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index 40d9c20..8738bf0 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -34,7 +34,6 @@ import java.io.Writer;
 import java.security.PrivilegedAction;
 import java.util.Arrays;
 import java.util.Comparator;
-import java.util.Date;
 import java.util.TreeSet;
 
 import org.apache.hadoop.security.UserGroupInformation;
@@ -193,6 +192,11 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 		return ret;
 	}
 
+	@Override
+	public boolean isEmpty() {
+		return mDispatcherThread == null || mDispatcherThread.isIdle();
+	}
+
 	private synchronized void openFile() {
 		LogLog.debug("==> LocalFileLogBuffer.openFile()");
 
@@ -283,7 +287,7 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 		}
 	}
 
-	public OutputStreamWriter createWriter(OutputStream os ) {
+	private OutputStreamWriter createWriter(OutputStream os ) {
 	    OutputStreamWriter writer = null;
 
 	    if(os != null) {
@@ -359,6 +363,12 @@ class DestinationDispatcherThread<T> extends Thread {
 		mStopThread = true;
 	}
 
+	public boolean isIdle() {
+		synchronized(mCompletedLogfiles) {
+			return mCompletedLogfiles.isEmpty() && mCurrentLogfile == null;
+		}
+	}
+
 	@Override
 	public void run() {
 		UserGroupInformation loginUser = null;

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
index ce67e01..d8ff10a 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LogBuffer.java
@@ -26,5 +26,7 @@ public interface LogBuffer<T> {
 
 	boolean isAvailable();
 
+	public boolean isEmpty();
+
 	public boolean add(T log);
 }

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
index ffe84c5..6ab78ea 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
@@ -13,7 +13,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
 	}
 
 	public void init(Map<String, String> properties) {
-		String encoding                               = properties.get("encoding");
+		String encoding                                = properties.get("encoding");
 
 		String hdfsDestinationDirectory                = properties.get("destination.directroy");
 		String hdfsDestinationFile                     = properties.get("destination.file");
@@ -50,5 +50,20 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
 		setBufferAndDestination(mLocalFileBuffer, mHdfsDestination);
 	}
 
+	@Override
+	public void waitToComplete() {
+		while(getBuffer() != null && !getBuffer().isEmpty()) {
+			sleep(1000);
+		}
+	}
+	
+	private static void sleep(int timeInMs) {
+		try {
+			Thread.sleep(timeInMs);
+		} catch(InterruptedException excp) {
+			
+		}
+	}
+
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java b/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
index aca18fd..dd7e035 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/test/TestEvents.java
@@ -96,6 +96,10 @@ public class TestEvents {
                 if(i != 0 && ((i % 100) == 0))
                     Thread.sleep(100);
         	}
+
+            provider.waitToComplete();
+
+            provider.stop();
         } catch(Exception excp) {
             LOG.info(excp.getLocalizedMessage());
         	excp.printStackTrace();

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/ea4df438/lookup-client/src/scripts/xasecure-audit.properties
----------------------------------------------------------------------
diff --git a/lookup-client/src/scripts/xasecure-audit.properties b/lookup-client/src/scripts/xasecure-audit.properties
index 9989c00..904b19c 100644
--- a/lookup-client/src/scripts/xasecure-audit.properties
+++ b/lookup-client/src/scripts/xasecure-audit.properties
@@ -1,16 +1,33 @@
-xasecure.audit.jpa.javax.persistence.jdbc.url=jdbc:mysql://localhost:3306/xasecure
-xasecure.audit.jpa.javax.persistence.jdbc.user=xalogger
-xasecure.audit.jpa.javax.persistence.jdbc.password=xalogger
-xasecure.audit.jpa.javax.persistence.jdbc.driver=com.mysql.jdbc.Driver
-	
 xasecure.audit.is.enabled=true
+
+
 xasecure.audit.log4j.is.enabled=false
 xasecure.audit.log4j.is.async=false
 xasecure.audit.log4j.async.max.queue.size=100000
 xasecure.audit.log4j.async.max.flush.interval.ms=30000
-xasecure.audit.db.is.enabled=true
+
+
+xasecure.audit.db.is.enabled=false
 xasecure.audit.db.is.async=true
 xasecure.audit.db.async.max.queue.size=102400
 xasecure.audit.db.async.resume.queue.size=92400
 xasecure.audit.db.async.max.flush.interval.ms=30000
 xasecure.audit.db.batch.size=100
+xasecure.audit.jpa.javax.persistence.jdbc.url=jdbc:mysql://localhost:3306/xasecure
+xasecure.audit.jpa.javax.persistence.jdbc.user=xalogger
+xasecure.audit.jpa.javax.persistence.jdbc.password=xalogger
+xasecure.audit.jpa.javax.persistence.jdbc.driver=com.mysql.jdbc.Driver
+
+
+xasecure.audit.hdfs.is.enabled=false
+xasecure.audit.hdfs.config.destination.directroy=hdfs://%hostname%:8020/tmp/audit/hdfs/%time:yyyyMMdd%
+xasecure.audit.hdfs.config.destination.file=%hostname%-audit.log
+xasecure.audit.hdfs.config.destination.flush.interval.seconds=900
+xasecure.audit.hdfs.config.destination.rollover.interval.seconds=86400
+xasecure.audit.hdfs.config.destination.open.retry.interval.seconds=60
+xasecure.audit.hdfs.config.local.buffer.directroy=/tmp/audit/hdfs
+xasecure.audit.hdfs.config.local.buffer.file=%time:yyyyMMdd-HHmm.ss%.log
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds=60
+xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds=600
+xasecure.audit.hdfs.config.local.archive.directroy=/tmp/audit/archive/hdfs
+xasecure.audit.hdfs.config.local.archive.max.file.count=10


Mime
View raw message