Return-Path: X-Original-To: apmail-argus-commits-archive@minotaur.apache.org Delivered-To: apmail-argus-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id E617F11E49 for ; Tue, 23 Sep 2014 18:33:51 +0000 (UTC) Received: (qmail 10355 invoked by uid 500); 23 Sep 2014 18:33:51 -0000 Delivered-To: apmail-argus-commits-archive@argus.apache.org Received: (qmail 10335 invoked by uid 500); 23 Sep 2014 18:33:51 -0000 Mailing-List: contact commits-help@argus.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@argus.incubator.apache.org Delivered-To: mailing list commits@argus.incubator.apache.org Received: (qmail 10325 invoked by uid 99); 23 Sep 2014 18:33:51 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 23 Sep 2014 18:33:51 +0000 X-ASF-Spam-Status: No, hits=-2000.8 required=5.0 tests=ALL_TRUSTED,RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Tue, 23 Sep 2014 18:33:49 +0000 Received: (qmail 7883 invoked by uid 99); 23 Sep 2014 18:33:28 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 23 Sep 2014 18:33:28 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id 9A511937DE5; Tue, 23 Sep 2014 18:33:28 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@argus.incubator.apache.org Date: Tue, 23 Sep 2014 18:33:28 -0000 Message-Id: <10b81161b51b498bb5948826d013726f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [1/5] git commit: ARGUS-5: added periodic flush to local and HDFS files; and configurable parameters for the same in xasecure-audit.xml and install.properties X-Virus-Checked: Checked by ClamAV on apache.org Repository: incubator-argus Updated Branches: refs/heads/master 6c11ec265 -> 647cdb7ee ARGUS-5: added periodic flush to local and HDFS files; and configurable parameters for the same in xasecure-audit.xml and install.properties Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/74d83939 Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/74d83939 Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/74d83939 Branch: refs/heads/master Commit: 74d83939558f6378ccb2cc2e4e6498cc980b1776 Parents: 81f7607 Author: mneethiraj Authored: Mon Sep 22 21:02:23 2014 -0700 Committer: mneethiraj Committed: Mon Sep 22 21:02:23 2014 -0700 ---------------------------------------------------------------------- .../audit/provider/LocalFileLogBuffer.java | 23 +++++++++++++++++++- .../audit/provider/hdfs/HdfsAuditProvider.java | 4 ++++ .../audit/provider/hdfs/HdfsLogDestination.java | 20 +++++++++++++++++ hbase-agent/conf/xasecure-audit-changes.cfg | 2 ++ hbase-agent/conf/xasecure-audit.xml | 10 +++++++++ hbase-agent/scripts/install.properties | 2 ++ hdfs-agent/conf/xasecure-audit-changes.cfg | 2 ++ hdfs-agent/conf/xasecure-audit.xml | 10 +++++++++ hdfs-agent/scripts/install.properties | 2 ++ hive-agent/conf/xasecure-audit-changes.cfg | 2 ++ hive-agent/conf/xasecure-audit.xml | 10 +++++++++ hive-agent/scripts/install.properties | 2 ++ knox-agent/conf/xasecure-audit-changes.cfg | 2 ++ knox-agent/conf/xasecure-audit.xml | 10 +++++++++ knox-agent/scripts/install.properties | 2 ++ storm-agent/conf/xasecure-audit-changes.cfg | 2 ++ storm-agent/conf/xasecure-audit.xml | 10 +++++++++ storm-agent/scripts/install.properties | 2 ++ 18 files changed, 116 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java index c6f2d6f..40d9c20 100644 --- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java +++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java @@ -34,6 +34,7 @@ import java.io.Writer; import java.security.PrivilegedAction; import java.util.Arrays; import java.util.Comparator; +import java.util.Date; import java.util.TreeSet; import org.apache.hadoop.security.UserGroupInformation; @@ -43,15 +44,17 @@ import org.apache.log4j.helpers.LogLog; public class LocalFileLogBuffer implements LogBuffer { private String mDirectory = null; private String mFile = null; + private int mFlushIntervalSeconds = 1 * 60; private String mEncoding = null; private boolean mIsAppend = true; - private int mRolloverIntervalSeconds = 600; + private int mRolloverIntervalSeconds = 10 * 60; private String mArchiveDirectory = null; private int mArchiveFileCount = 10; private Writer mWriter = null; private String mBufferFilename = null; private long mNextRolloverTime = 0; + private long mNextFlushTime = 0; private DestinationDispatcherThread mDispatcherThread = null; @@ -74,6 +77,14 @@ public class LocalFileLogBuffer implements LogBuffer { mFile = file; } + public int getFlushIntervalSeconds() { + return mFlushIntervalSeconds; + } + + public void setFlushIntervalSeconds(int flushIntervalSeconds) { + mFlushIntervalSeconds = flushIntervalSeconds; + } + public String getEncoding() { return mEncoding; } @@ -210,6 +221,8 @@ public class LocalFileLogBuffer implements LogBuffer { if(mWriter != null) { LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename); + + mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000); } else { LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename); @@ -259,6 +272,14 @@ public class LocalFileLogBuffer implements LogBuffer { rollover(); } else if(mWriter == null) { openFile(); + } else if(now > mNextFlushTime) { + try { + mWriter.flush(); + + mNextFlushTime = now + (mFlushIntervalSeconds * 1000); + } catch (IOException excp) { + LogLog.warn("LocalFileLogBuffer: failed to flush", excp); + } } } http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java index e8b3922..ffe84c5 100644 --- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java +++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsAuditProvider.java @@ -17,11 +17,13 @@ public class HdfsAuditProvider extends BufferedAuditProvider { String hdfsDestinationDirectory = properties.get("destination.directroy"); String hdfsDestinationFile = properties.get("destination.file"); + int hdfsDestinationFlushIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.flush.interval.seconds"), 15 * 60); int hdfsDestinationRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.rollover.interval.seconds"), 24 * 60 * 60); int hdfsDestinationOpenRetryIntervalSeconds = MiscUtil.parseInteger(properties.get("destination.open.retry.interval.seconds"), 60); String localFileBufferDirectory = properties.get("local.buffer.directroy"); String localFileBufferFile = properties.get("local.buffer.file"); + int localFileBufferFlushIntervalSeconds = MiscUtil.parseInteger(properties.get("local.buffer.flush.interval.seconds"), 1 * 60); int localFileBufferRolloverIntervalSeconds = MiscUtil.parseInteger(properties.get("local.buffer.rollover.interval.seconds"), 10 * 60); String localFileBufferArchiveDirectory = properties.get("local.archive.directroy"); int localFileBufferArchiveFileCount = MiscUtil.parseInteger(properties.get("local.archive.max.file.count"), 10); @@ -30,6 +32,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider { mHdfsDestination.setDirectory(hdfsDestinationDirectory); mHdfsDestination.setFile(hdfsDestinationFile); + mHdfsDestination.setFlushIntervalSeconds(hdfsDestinationFlushIntervalSeconds); mHdfsDestination.setEncoding(encoding); mHdfsDestination.setRolloverIntervalSeconds(hdfsDestinationRolloverIntervalSeconds); mHdfsDestination.setOpenRetryIntervalSeconds(hdfsDestinationOpenRetryIntervalSeconds); @@ -38,6 +41,7 @@ public class HdfsAuditProvider extends BufferedAuditProvider { mLocalFileBuffer.setDirectory(localFileBufferDirectory); mLocalFileBuffer.setFile(localFileBufferFile); + mLocalFileBuffer.setFlushIntervalSeconds(localFileBufferFlushIntervalSeconds); mLocalFileBuffer.setEncoding(encoding); mLocalFileBuffer.setRolloverIntervalSeconds(localFileBufferRolloverIntervalSeconds); mLocalFileBuffer.setArchiveDirectory(localFileBufferArchiveDirectory); http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java index 0b6dd6e..42119f8 100644 --- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java +++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java @@ -25,6 +25,7 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.net.URI; +import java.util.Date; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; @@ -38,6 +39,7 @@ import com.xasecure.audit.provider.MiscUtil; public class HdfsLogDestination implements LogDestination { private String mDirectory = null; private String mFile = null; + private int mFlushIntervalSeconds = 1 * 60; private String mEncoding = null; private boolean mIsAppend = true; private int mRolloverIntervalSeconds = 24 * 60 * 60; @@ -46,6 +48,7 @@ public class HdfsLogDestination implements LogDestination { private OutputStreamWriter mWriter = null; private String mHdfsFilename = null; private long mNextRolloverTime = 0; + private long mNextFlushTime = 0; private long mLastOpenFailedTime = 0; private boolean mIsStopInProgress = false; @@ -68,6 +71,14 @@ public class HdfsLogDestination implements LogDestination { this.mFile = file; } + public int getFlushIntervalSeconds() { + return mFlushIntervalSeconds; + } + + public void setFlushIntervalSeconds(int flushIntervalSeconds) { + mFlushIntervalSeconds = flushIntervalSeconds; + } + public String getEncoding() { return mEncoding; } @@ -224,6 +235,7 @@ public class HdfsLogDestination implements LogDestination { if(mWriter != null) { LogLog.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename); + mNextFlushTime = System.currentTimeMillis() + (mFlushIntervalSeconds * 1000); mLastOpenFailedTime = 0; } else { LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename); @@ -275,6 +287,14 @@ public class HdfsLogDestination implements LogDestination { } } else if(now > mNextRolloverTime) { rollover(); + } else if(now > mNextFlushTime) { + try { + mWriter.flush(); + + mNextFlushTime = now + (mFlushIntervalSeconds * 1000); + } catch (IOException excp) { + LogLog.warn("HdfsLogDestination: failed to flush", excp); + } } } http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/conf/xasecure-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hbase-agent/conf/xasecure-audit-changes.cfg b/hbase-agent/conf/xasecure-audit-changes.cfg index 66beb58..48a8cf5 100644 --- a/hbase-agent/conf/xasecure-audit-changes.cfg +++ b/hbase-agent/conf/xasecure-audit-changes.cfg @@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/conf/xasecure-audit.xml ---------------------------------------------------------------------- diff --git a/hbase-agent/conf/xasecure-audit.xml b/hbase-agent/conf/xasecure-audit.xml index 93f06d1..c57d580 100644 --- a/hbase-agent/conf/xasecure-audit.xml +++ b/hbase-agent/conf/xasecure-audit.xml @@ -131,6 +131,11 @@ + xasecure.audit.hdfs.config.destination.flush.interval.seconds + 900 + + + xasecure.audit.hdfs.config.destination.rollover.interval.seconds 86400 @@ -151,6 +156,11 @@ + xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds + 60 + + + xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds 600 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hbase-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties index 337b3f5..2e227dc 100644 --- a/hbase-agent/scripts/install.properties +++ b/hbase-agent/scripts/install.properties @@ -114,10 +114,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver XAAUDIT.HDFS.IS_ENABLED=false XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hbase/%time:yyyyMMdd% XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hbase XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hbase XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/conf/xasecure-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hdfs-agent/conf/xasecure-audit-changes.cfg b/hdfs-agent/conf/xasecure-audit-changes.cfg index 66beb58..48a8cf5 100644 --- a/hdfs-agent/conf/xasecure-audit-changes.cfg +++ b/hdfs-agent/conf/xasecure-audit-changes.cfg @@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/conf/xasecure-audit.xml ---------------------------------------------------------------------- diff --git a/hdfs-agent/conf/xasecure-audit.xml b/hdfs-agent/conf/xasecure-audit.xml index e2b8f9d..e7c23d4 100644 --- a/hdfs-agent/conf/xasecure-audit.xml +++ b/hdfs-agent/conf/xasecure-audit.xml @@ -112,6 +112,11 @@ + xasecure.audit.hdfs.config.destination.flush.interval.seconds + 900 + + + xasecure.audit.hdfs.config.destination.rollover.interval.seconds 86400 @@ -132,6 +137,11 @@ + xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds + 60 + + + xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds 600 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hdfs-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties index 5cfa476..d8b6680 100644 --- a/hdfs-agent/scripts/install.properties +++ b/hdfs-agent/scripts/install.properties @@ -107,10 +107,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver XAAUDIT.HDFS.IS_ENABLED=false XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hdfs/%time:yyyyMMdd% XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hdfs XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hdfs XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/conf/xasecure-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hive-agent/conf/xasecure-audit-changes.cfg b/hive-agent/conf/xasecure-audit-changes.cfg index 0ae94fb..e8547c2 100644 --- a/hive-agent/conf/xasecure-audit-changes.cfg +++ b/hive-agent/conf/xasecure-audit-changes.cfg @@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/conf/xasecure-audit.xml ---------------------------------------------------------------------- diff --git a/hive-agent/conf/xasecure-audit.xml b/hive-agent/conf/xasecure-audit.xml index 319ae09..35c4c8c 100644 --- a/hive-agent/conf/xasecure-audit.xml +++ b/hive-agent/conf/xasecure-audit.xml @@ -131,6 +131,11 @@ + xasecure.audit.hdfs.config.destination.flush.interval.seconds + 900 + + + xasecure.audit.hdfs.config.destination.rollover.interval.seconds 86400 @@ -151,6 +156,11 @@ + xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds + 60 + + + xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds 600 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/hive-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties index 4cbdde0..1c4a232 100644 --- a/hive-agent/scripts/install.properties +++ b/hive-agent/scripts/install.properties @@ -114,10 +114,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver XAAUDIT.HDFS.IS_ENABLED=false XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/hive/%time:yyyyMMdd% XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/hive XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/hive XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/conf/xasecure-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/knox-agent/conf/xasecure-audit-changes.cfg b/knox-agent/conf/xasecure-audit-changes.cfg index 9e0340e..02e6fb1 100644 --- a/knox-agent/conf/xasecure-audit-changes.cfg +++ b/knox-agent/conf/xasecure-audit-changes.cfg @@ -8,10 +8,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/conf/xasecure-audit.xml ---------------------------------------------------------------------- diff --git a/knox-agent/conf/xasecure-audit.xml b/knox-agent/conf/xasecure-audit.xml index 4a7303e..c861abb 100644 --- a/knox-agent/conf/xasecure-audit.xml +++ b/knox-agent/conf/xasecure-audit.xml @@ -126,6 +126,11 @@ + xasecure.audit.hdfs.config.destination.flush.interval.seconds + 900 + + + xasecure.audit.hdfs.config.destination.rollover.interval.seconds 86400 @@ -146,6 +151,11 @@ + xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds + 60 + + + xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds 600 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/knox-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties index 1f0e01e..c93e844 100644 --- a/knox-agent/scripts/install.properties +++ b/knox-agent/scripts/install.properties @@ -100,10 +100,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver XAAUDIT.HDFS.IS_ENABLED=false XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/knox/%time:yyyyMMdd% XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/knox XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/knox XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/conf/xasecure-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/storm-agent/conf/xasecure-audit-changes.cfg b/storm-agent/conf/xasecure-audit-changes.cfg index 68a2484..bfbc90f 100644 --- a/storm-agent/conf/xasecure-audit-changes.cfg +++ b/storm-agent/conf/xasecure-audit-changes.cfg @@ -9,10 +9,12 @@ xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT.DB.JDBC_DRIVER% xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists xasecure.audit.hdfs.config.destination.directroy %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.directroy %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directroy %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/conf/xasecure-audit.xml ---------------------------------------------------------------------- diff --git a/storm-agent/conf/xasecure-audit.xml b/storm-agent/conf/xasecure-audit.xml index f8c07d2..acac19b 100644 --- a/storm-agent/conf/xasecure-audit.xml +++ b/storm-agent/conf/xasecure-audit.xml @@ -131,6 +131,11 @@ + xasecure.audit.hdfs.config.destination.flush.interval.seconds + 900 + + + xasecure.audit.hdfs.config.destination.rollover.interval.seconds 86400 @@ -151,6 +156,11 @@ + xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds + 60 + + + xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds 600 http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/74d83939/storm-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/storm-agent/scripts/install.properties b/storm-agent/scripts/install.properties index c1e3e44..2d28d5d 100644 --- a/storm-agent/scripts/install.properties +++ b/storm-agent/scripts/install.properties @@ -104,10 +104,12 @@ XAAUDIT.DB.JDBC_DRIVER=com.mysql.jdbc.Driver XAAUDIT.HDFS.IS_ENABLED=false XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://localhost:8020/audit/storm/%time:yyyyMMdd% XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/tmp/logs/storm XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/tmp/logs/archive/storm XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10