ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [1/5] git commit: ARGUS-5: added periodic flush to local and HDFS files; and configurable parameters for the same in xasecure-audit.xml and install.properties
Date Tue, 23 Sep 2014 18:33:28 GMT
Repository: incubator-argus
Updated Branches:
  refs/heads/master 6c11ec265 -> 647cdb7ee


ARGUS-5: added periodic flush to local and HDFS files; and configurable
parameters for the same in xasecure-audit.xml and install.properties

Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/74d83939
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/74d83939
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/74d83939

Branch: refs/heads/master
Commit: 74d83939558f6378ccb2cc2e4e6498cc980b1776
Parents: 81f7607
Author: mneethiraj <mneethiraj@hortonworks.com>
Authored: Mon Sep 22 21:02:23 2014 -0700
Committer: mneethiraj <mneethiraj@hortonworks.com>
Committed: Mon Sep 22 21:02:23 2014 -0700

----------------------------------------------------------------------
 .../audit/provider/LocalFileLogBuffer.java      | 23 +++++++++++++++++++-
 .../audit/provider/hdfs/HdfsAuditProvider.java  |  4 ++++
 .../audit/provider/hdfs/HdfsLogDestination.java | 20 +++++++++++++++++
 hbase-agent/conf/xasecure-audit-changes.cfg     |  2 ++
 hbase-agent/conf/xasecure-audit.xml             | 10 +++++++++
 hbase-agent/scripts/install.properties          |  2 ++
 hdfs-agent/conf/xasecure-audit-changes.cfg      |  2 ++
 hdfs-agent/conf/xasecure-audit.xml              | 10 +++++++++
 hdfs-agent/scripts/install.properties           |  2 ++
 hive-agent/conf/xasecure-audit-changes.cfg      |  2 ++
 hive-agent/conf/xasecure-audit.xml              | 10 +++++++++
 hive-agent/scripts/install.properties           |  2 ++
 knox-agent/conf/xasecure-audit-changes.cfg      |  2 ++
 knox-agent/conf/xasecure-audit.xml              | 10 +++++++++
 knox-agent/scripts/install.properties           |  2 ++
 storm-agent/conf/xasecure-audit-changes.cfg     |  2 ++
 storm-agent/conf/xasecure-audit.xml             | 10 +++++++++
 storm-agent/scripts/install.properties          |  2 ++
 18 files changed, 116 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index c6f2d6f..40d9c20 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -34,6 +34,7 @@ import java.io.Writer;
 import java.security.PrivilegedAction;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.Date;
 import java.util.TreeSet;
 
 import org.apache.hadoop.security.UserGroupInformation;
@@ -43,15 +44,17 @@ import org.apache.log4j.helpers.LogLog;
 public class LocalFileLogBuffer<T> implements LogBuffer<T> {
 	private String  mDirectory               = null;
 	private String  mFile                    = null;
+	private int     mFlushIntervalSeconds    = 1 * 60;
 	private String  mEncoding                = null;
 	private boolean mIsAppend                = true;
-	private int     mRolloverIntervalSeconds = 600;
+	private int     mRolloverIntervalSeconds = 10 * 60;
 	private String  mArchiveDirectory        = null;
 	private int     mArchiveFileCount        = 10;
 
 	private Writer mWriter           = null;
 	private String mBufferFilename   = null;
 	private long   mNextRolloverTime = 0;
+	private long   mNextFlushTime    = 0;
 
 	private DestinationDispatcherThread<T> mDispatcherThread = null;
 	
@@ -74,6 +77,14 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 		mFile = file;
 	}
 
+	public int getFlushIntervalSeconds() {
+		return mFlushIntervalSeconds;
+	}
+
+	public void setFlushIntervalSeconds(int flushIntervalSeconds) {
+		mFlushIntervalSeconds = flushIntervalSeconds;
+	}
+
 	public String getEncoding() {
 		return mEncoding;
 	}
@@ -210,6 +221,8 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 
 		if(mWriter != null) {
 			LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename);
+
+			mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000);
 		} else {
 			LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename);
 
@@ -259,6 +272,14 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 			rollover();
 		} else  if(mWriter == null) {
 			openFile();
+		} else if(now > mNextFlushTime) {
+			try {
+				mWriter.flush();
+
+				mNextFlushTime = now + (mFlushIntervalSeconds * 1000);
+			} catch (IOException excp) {
+				LogLog.warn("LocalFileLogBuffer: failed to flush", excp);
+			}
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
index e8b3922..ffe84c5 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java
@@ -17,11 +17,13 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
 
 		String hdfsDestinationDirectory                = properties.get("destination.directroy");
 		String hdfsDestinationFile                     = properties.get("destination.file");
+		int    hdfsDestinationFlushIntervalSeconds     = MiscUtil.parseInteger(properties.get("destination.flush.interval.seconds"),
15 * 60);
 		int    hdfsDestinationRolloverIntervalSeconds  = MiscUtil.parseInteger(properties.get("destination.rollover.interval.seconds"),
24 * 60 * 60);
 		int    hdfsDestinationOpenRetryIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.open.retry.interval.seconds"),
60);
 
 		String localFileBufferDirectory               = properties.get("local.buffer.directroy");
 		String localFileBufferFile                    = properties.get("local.buffer.file");
+		int    localFileBufferFlushIntervalSeconds    = MiscUtil.parseInteger(properties.get("local.buffer.flush.interval.seconds"),
1 * 60);
 		int    localFileBufferRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("local.buffer.rollover.interval.seconds"),
10 * 60);
 		String localFileBufferArchiveDirectory        = properties.get("local.archive.directroy");
 		int    localFileBufferArchiveFileCount        = MiscUtil.parseInteger(properties.get("local.archive.max.file.count"),
10);
@@ -30,6 +32,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
 
 		mHdfsDestination.setDirectory(hdfsDestinationDirectory);
 		mHdfsDestination.setFile(hdfsDestinationFile);
+		mHdfsDestination.setFlushIntervalSeconds(hdfsDestinationFlushIntervalSeconds);
 		mHdfsDestination.setEncoding(encoding);
 		mHdfsDestination.setRolloverIntervalSeconds(hdfsDestinationRolloverIntervalSeconds);
 		mHdfsDestination.setOpenRetryIntervalSeconds(hdfsDestinationOpenRetryIntervalSeconds);
@@ -38,6 +41,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider {
 
 		mLocalFileBuffer.setDirectory(localFileBufferDirectory);
 		mLocalFileBuffer.setFile(localFileBufferFile);
+		mLocalFileBuffer.setFlushIntervalSeconds(localFileBufferFlushIntervalSeconds);
 		mLocalFileBuffer.setEncoding(encoding);
 		mLocalFileBuffer.setRolloverIntervalSeconds(localFileBufferRolloverIntervalSeconds);
 		mLocalFileBuffer.setArchiveDirectory(localFileBufferArchiveDirectory);

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
index 0b6dd6e..42119f8 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -25,6 +25,7 @@ import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
 import java.net.URI;
+import java.util.Date;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -38,6 +39,7 @@ import com.xasecure.audit.provider.MiscUtil;
 public class HdfsLogDestination<T> implements LogDestination<T> {
 	private String  mDirectory                = null;
 	private String  mFile                     = null;
+	private int     mFlushIntervalSeconds     = 1 * 60;
 	private String  mEncoding                 = null;
 	private boolean mIsAppend                 = true;
 	private int     mRolloverIntervalSeconds  = 24 * 60 * 60;
@@ -46,6 +48,7 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 	private OutputStreamWriter mWriter             = null; 
 	private String             mHdfsFilename       = null;
 	private long               mNextRolloverTime   = 0;
+	private long               mNextFlushTime      = 0;
 	private long               mLastOpenFailedTime = 0;
 	private boolean            mIsStopInProgress   = false;
 
@@ -68,6 +71,14 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 		this.mFile = file;
 	}
 
+	public int getFlushIntervalSeconds() {
+		return mFlushIntervalSeconds;
+	}
+
+	public void setFlushIntervalSeconds(int flushIntervalSeconds) {
+		mFlushIntervalSeconds = flushIntervalSeconds;
+	}
+
 	public String getEncoding() {
 		return mEncoding;
 	}
@@ -224,6 +235,7 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 		if(mWriter != null) {
 			LogLog.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename);
 
+			mNextFlushTime      = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000);
 			mLastOpenFailedTime = 0;
 		} else {
 			LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename);
@@ -275,6 +287,14 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 			}
 		} else  if(now > mNextRolloverTime) {
 			rollover();
+		} else if(now > mNextFlushTime) {
+			try {
+				mWriter.flush();
+
+				mNextFlushTime = now + (mFlushIntervalSeconds * 1000);
+			} catch (IOException excp) {
+				LogLog.warn("HdfsLogDestination: failed to flush", excp);
+			}
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/xasecure-audit-changes.cfg b/hbase-agent/conf/xasecure-audit-changes.cfg
index 66beb58..48a8cf5 100644
--- a/hbase-agent/conf/xasecure-audit-changes.cfg
+++ b/hbase-agent/conf/xasecure-audit-changes.cfg
@@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver	%XAAUDIT.DB.JDBC_DRIVER%
 xasecure.audit.hdfs.is.enabled                                     %XAAUDIT.HDFS.IS_ENABLED%
                              mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.directroy                   %XAAUDIT.HDFS.DESTINATION_DIRECTORY%
                   mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.file                        %XAAUDIT.HDFS.DESTINTATION_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds      %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.rollover.interval.seconds   %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS%
mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.directroy                  %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY%
                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.file                       %XAAUDIT.HDFS.LOCAL_BUFFER_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directroy                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%
                 mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%
            mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/xasecure-audit.xml b/hbase-agent/conf/xasecure-audit.xml
index 93f06d1..c57d580 100644
--- a/hbase-agent/conf/xasecure-audit.xml
+++ b/hbase-agent/conf/xasecure-audit.xml
@@ -131,6 +131,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+		<value>900</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
 		<value>86400</value>
 	</property>	
@@ -151,6 +156,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+		<value>60</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
 		<value>600</value>
 	</property>	

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index 337b3f5..2e227dc 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -114,10 +114,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
 XAAUDIT.HDFS.IS_ENABLED=false
 XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hbase/%time:yyyyMMdd%
 XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hbase
 XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hbase
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/xasecure-audit-changes.cfg b/hdfs-agent/conf/xasecure-audit-changes.cfg
index 66beb58..48a8cf5 100644
--- a/hdfs-agent/conf/xasecure-audit-changes.cfg
+++ b/hdfs-agent/conf/xasecure-audit-changes.cfg
@@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver	%XAAUDIT.DB.JDBC_DRIVER%
 xasecure.audit.hdfs.is.enabled                                     %XAAUDIT.HDFS.IS_ENABLED%
                              mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.directroy                   %XAAUDIT.HDFS.DESTINATION_DIRECTORY%
                   mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.file                        %XAAUDIT.HDFS.DESTINTATION_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds      %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.rollover.interval.seconds   %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS%
mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.directroy                  %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY%
                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.file                       %XAAUDIT.HDFS.LOCAL_BUFFER_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directroy                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%
                 mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%
            mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/xasecure-audit.xml b/hdfs-agent/conf/xasecure-audit.xml
index e2b8f9d..e7c23d4 100644
--- a/hdfs-agent/conf/xasecure-audit.xml
+++ b/hdfs-agent/conf/xasecure-audit.xml
@@ -112,6 +112,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+		<value>900</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
 		<value>86400</value>
 	</property>	
@@ -132,6 +137,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+		<value>60</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
 		<value>600</value>
 	</property>	

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 5cfa476..d8b6680 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -107,10 +107,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
 XAAUDIT.HDFS.IS_ENABLED=false
 XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hdfs/%time:yyyyMMdd%
 XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hdfs
 XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hdfs
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hive-agent/conf/xasecure-audit-changes.cfg b/hive-agent/conf/xasecure-audit-changes.cfg
index 0ae94fb..e8547c2 100644
--- a/hive-agent/conf/xasecure-audit-changes.cfg
+++ b/hive-agent/conf/xasecure-audit-changes.cfg
@@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver	%XAAUDIT.DB.JDBC_DRIVER%
 xasecure.audit.hdfs.is.enabled                                     %XAAUDIT.HDFS.IS_ENABLED%
                              mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.directroy                   %XAAUDIT.HDFS.DESTINATION_DIRECTORY%
                   mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.file                        %XAAUDIT.HDFS.DESTINTATION_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds      %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.rollover.interval.seconds   %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS%
mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.directroy                  %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY%
                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.file                       %XAAUDIT.HDFS.LOCAL_BUFFER_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directroy                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%
                 mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%
            mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/xasecure-audit.xml b/hive-agent/conf/xasecure-audit.xml
index 319ae09..35c4c8c 100644
--- a/hive-agent/conf/xasecure-audit.xml
+++ b/hive-agent/conf/xasecure-audit.xml
@@ -131,6 +131,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+		<value>900</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
 		<value>86400</value>
 	</property>	
@@ -151,6 +156,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+		<value>60</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
 		<value>600</value>
 	</property>	

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index 4cbdde0..1c4a232 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -114,10 +114,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
 XAAUDIT.HDFS.IS_ENABLED=false
 XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hive/%time:yyyyMMdd%
 XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hive
 XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hive
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/knox-agent/conf/xasecure-audit-changes.cfg b/knox-agent/conf/xasecure-audit-changes.cfg
index 9e0340e..02e6fb1 100644
--- a/knox-agent/conf/xasecure-audit-changes.cfg
+++ b/knox-agent/conf/xasecure-audit-changes.cfg
@@ -8,10 +8,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver	%XAAUDIT.DB.JDBC_DRIVER%
 xasecure.audit.hdfs.is.enabled                                     %XAAUDIT.HDFS.IS_ENABLED%
                              mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.directroy                   %XAAUDIT.HDFS.DESTINATION_DIRECTORY%
                   mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.file                        %XAAUDIT.HDFS.DESTINTATION_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds      %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.rollover.interval.seconds   %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS%
mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.directroy                  %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY%
                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.file                       %XAAUDIT.HDFS.LOCAL_BUFFER_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directroy                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%
                 mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%
            mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/xasecure-audit.xml b/knox-agent/conf/xasecure-audit.xml
index 4a7303e..c861abb 100644
--- a/knox-agent/conf/xasecure-audit.xml
+++ b/knox-agent/conf/xasecure-audit.xml
@@ -126,6 +126,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+		<value>900</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
 		<value>86400</value>
 	</property>	
@@ -146,6 +151,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+		<value>60</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
 		<value>600</value>
 	</property>	

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index 1f0e01e..c93e844 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -100,10 +100,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
 XAAUDIT.HDFS.IS_ENABLED=false
 XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/knox/%time:yyyyMMdd%
 XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/knox
 XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/knox
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/conf/xasecure-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/storm-agent/conf/xasecure-audit-changes.cfg b/storm-agent/conf/xasecure-audit-changes.cfg
index 68a2484..bfbc90f 100644
--- a/storm-agent/conf/xasecure-audit-changes.cfg
+++ b/storm-agent/conf/xasecure-audit-changes.cfg
@@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver	%XAAUDIT.DB.JDBC_DRIVER%
 xasecure.audit.hdfs.is.enabled                                     %XAAUDIT.HDFS.IS_ENABLED%
                              mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.directroy                   %XAAUDIT.HDFS.DESTINATION_DIRECTORY%
                   mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.file                        %XAAUDIT.HDFS.DESTINTATION_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.destination.flush.interval.seconds      %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.rollover.interval.seconds   %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS%
mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.directroy                  %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY%
                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.file                       %XAAUDIT.HDFS.LOCAL_BUFFER_FILE%
                       mod create-if-not-exists
+xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS%
     mod create-if-not-exists
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%
  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directroy                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%
                 mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%
            mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/conf/xasecure-audit.xml
----------------------------------------------------------------------
diff --git a/storm-agent/conf/xasecure-audit.xml b/storm-agent/conf/xasecure-audit.xml
index f8c07d2..acac19b 100644
--- a/storm-agent/conf/xasecure-audit.xml
+++ b/storm-agent/conf/xasecure-audit.xml
@@ -131,6 +131,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name>
+		<value>900</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name>
 		<value>86400</value>
 	</property>	
@@ -151,6 +156,11 @@
 	</property>	
 
 	<property>
+		<name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name>
+		<value>60</value>
+	</property>	
+
+	<property>
 		<name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name>
 		<value>600</value>
 	</property>	

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties
index c1e3e44..2d28d5d 100644
--- a/storm-agent/scripts/install.properties
+++ b/storm-agent/scripts/install.properties
@@ -104,10 +104,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver
 XAAUDIT.HDFS.IS_ENABLED=false
 XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/storm/%time:yyyyMMdd%
 XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log
+XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900
 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400
 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/storm
 XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log
+XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/storm
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10


Mime
View raw message