Return-Path: X-Original-To: apmail-ranger-commits-archive@www.apache.org Delivered-To: apmail-ranger-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 69AF81899E for ; Tue, 2 Jun 2015 04:45:30 +0000 (UTC) Received: (qmail 61440 invoked by uid 500); 2 Jun 2015 04:45:30 -0000 Delivered-To: apmail-ranger-commits-archive@ranger.apache.org Received: (qmail 61414 invoked by uid 500); 2 Jun 2015 04:45:30 -0000 Mailing-List: contact commits-help@ranger.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ranger.incubator.apache.org Delivered-To: mailing list commits@ranger.incubator.apache.org Received: (qmail 61405 invoked by uid 99); 2 Jun 2015 04:45:29 -0000 Received: from Unknown (HELO spamd3-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Jun 2015 04:45:29 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd3-us-west.apache.org (ASF Mail Server at spamd3-us-west.apache.org) with ESMTP id 7F1B0183A2E for ; Tue, 2 Jun 2015 04:45:29 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd3-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: 1.77 X-Spam-Level: * X-Spam-Status: No, score=1.77 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, T_RP_MATCHES_RCVD=-0.01] autolearn=disabled Received: from mx1-us-east.apache.org ([10.40.0.8]) by localhost (spamd3-us-west.apache.org [10.40.0.10]) (amavisd-new, port 10024) with ESMTP id cfXNWr5B-UqN for ; Tue, 2 Jun 2015 04:45:26 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-us-east.apache.org (ASF Mail Server at mx1-us-east.apache.org) with SMTP id 1374047BE6 for ; Tue, 2 Jun 2015 04:45:25 +0000 (UTC) Received: (qmail 61399 invoked by uid 99); 2 Jun 2015 04:45:25 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 02 Jun 2015 04:45:25 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D5F5AE0593; Tue, 2 Jun 2015 04:45:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: bosco@apache.org To: commits@ranger.incubator.apache.org Message-Id: <4fb29ad2d66b42b393edac4465221a4c@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: incubator-ranger git commit: RANGER-397 Incorporate review comments Date: Tue, 2 Jun 2015 04:45:23 +0000 (UTC) Repository: incubator-ranger Updated Branches: refs/heads/master c52641711 -> 8469e643d RANGER-397 Incorporate review comments Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/8469e643 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/8469e643 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/8469e643 Branch: refs/heads/master Commit: 8469e643da3a81312aff58dc74673ed48e440f95 Parents: c526417 Author: Don Bosco Durai Authored: Mon Jun 1 18:02:42 2015 -0700 Committer: Don Bosco Durai Committed: Mon Jun 1 18:02:42 2015 -0700 ---------------------------------------------------------------------- .../audit/destination/DBAuditDestination.java | 3 ++- .../audit/destination/FileAuditDestination.java | 3 ++- .../audit/destination/HDFSAuditDestination.java | 7 ++++-- .../destination/Log4JAuditDestination.java | 15 ++++++++--- .../audit/destination/SolrAuditDestination.java | 8 +++++- .../ranger/audit/provider/BaseAuditHandler.java | 14 +++++++++-- .../audit/provider/MultiDestAuditProvider.java | 21 +++++++++++++--- .../ranger/audit/queue/AuditBatchQueue.java | 5 ++-- .../apache/ranger/audit/queue/AuditQueue.java | 26 +++++++++++++++----- 9 files changed, 79 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java index 6825637..3d31c06 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java @@ -78,7 +78,7 @@ public class DBAuditDestination extends AuditDestination { @Override public boolean log(Collection events) { boolean retValue = false; - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); if (beginTransaction()) { @@ -210,6 +210,7 @@ public class DBAuditDestination extends AuditDestination { entityManagerFactory = null; daoManager = null; } + logStatus(); } private EntityManager getEntityManager() { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java index 77c8f61..c6cd8b2 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java @@ -106,7 +106,7 @@ public class FileAuditDestination extends AuditDestination { @Override synchronized public boolean logJSON(Collection events) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); if (isStopped) { @@ -183,6 +183,7 @@ public class FileAuditDestination extends AuditDestination { } logWriter = null; } + logStatus(); } // Helper methods in this class http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java index b63fa40..e79e42d 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java @@ -105,7 +105,7 @@ public class HDFSAuditDestination extends AuditDestination { @Override synchronized public boolean logJSON(Collection events) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); if (!initDone) { @@ -142,8 +142,9 @@ public class HDFSAuditDestination extends AuditDestination { @Override public boolean log(Collection events) { if (isStopped) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); + addDeferredCount(events.size()); logError("log() called after stop was requested. name=" + getName()); return false; } @@ -153,6 +154,7 @@ public class HDFSAuditDestination extends AuditDestination { jsonList.add(MiscUtil.stringify(event)); } catch (Throwable t) { logger.error("Error converting to JSON. event=" + event); + addTotalCount(1); addFailedCount(1); logFailedEvent(event); } @@ -185,6 +187,7 @@ public class HDFSAuditDestination extends AuditDestination { } logWriter = null; } + logStatus(); } // Helper methods in this class http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java index b57cd86..9521a4a 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java @@ -59,10 +59,17 @@ public class Log4JAuditDestination extends AuditDestination { + ", loggerName=" + loggerName); } + + @Override + public void stop() { + super.stop(); + logStatus(); + } + @Override public boolean log(AuditEventBase event) { if (!auditLogger.isInfoEnabled()) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(1); return true; } @@ -77,7 +84,7 @@ public class Log4JAuditDestination extends AuditDestination { @Override public boolean log(Collection events) { if (!auditLogger.isInfoEnabled()) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); return true; } @@ -90,7 +97,7 @@ public class Log4JAuditDestination extends AuditDestination { @Override public boolean logJSON(String event) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(1); if (!auditLogger.isInfoEnabled()) { return true; @@ -106,7 +113,7 @@ public class Log4JAuditDestination extends AuditDestination { @Override public boolean logJSON(Collection events) { if (!auditLogger.isInfoEnabled()) { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); return true; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java index b43e573..213e4b2 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java @@ -55,6 +55,12 @@ public class SolrAuditDestination extends AuditDestination { super.init(props, propPrefix); connect(); } + + @Override + public void stop() { + super.stop(); + logStatus(); + } synchronized void connect() { if (solrClient == null) { @@ -103,7 +109,7 @@ public class SolrAuditDestination extends AuditDestination { @Override public boolean log(Collection events) { try { - logStatusIfRequired(true); + logStatusIfRequired(); addTotalCount(events.size()); if (solrClient == null) { http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java index ef85743..3859a7e 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java @@ -234,9 +234,17 @@ public abstract class BaseAuditHandler implements AuditHandler { return lastDeferredCount; } - public void logStatusIfRequired(boolean ifRequired) { + public void logStatusIfRequired() { long currTime = System.currentTimeMillis(); - if (!ifRequired || (currTime - lastStatusLogTime) > statusLogIntervalMS) { + if ((currTime - lastStatusLogTime) > statusLogIntervalMS) { + logStatus(); + } + } + + public void logStatus() { + try { + long currTime = System.currentTimeMillis(); + long diffTime = currTime - lastStatusLogTime; lastStatusLogTime = currTime; @@ -278,6 +286,8 @@ public abstract class BaseAuditHandler implements AuditHandler { + (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount) : ""); LOG.info(msg); + } catch (Throwable t) { + LOG.error("Error while printing stats. auditProvider=" + getName()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java index 9a5ff13..26108ca 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java @@ -33,13 +33,16 @@ public class MultiDestAuditProvider extends BaseAuditHandler { protected List mProviders = new ArrayList(); static final String DEFAULT_NAME = "multi_dest"; - + public MultiDestAuditProvider() { LOG.info("MultiDestAuditProvider: creating.."); setName(DEFAULT_NAME); } public MultiDestAuditProvider(AuditHandler provider) { + LOG.info("MultiDestAuditProvider(): provider=" + + (provider == null ? null : provider.getName())); + setName(DEFAULT_NAME); addAuditProvider(provider); } @@ -59,12 +62,22 @@ public class MultiDestAuditProvider extends BaseAuditHandler { } } - + @Override + public void setParentPath(String parentPath) { + super.setParentPath(parentPath); + for (AuditHandler provider : mProviders) { + if (provider instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; + baseAuditHander.setParentPath(getName()); + } + } + } + @Override public void setName(String name) { super.setName(name); for (AuditHandler provider : mProviders) { - if( provider instanceof BaseAuditHandler) { + if (provider instanceof BaseAuditHandler) { BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; baseAuditHander.setParentPath(getName()); } @@ -77,7 +90,7 @@ public class MultiDestAuditProvider extends BaseAuditHandler { + provider.getClass().getCanonicalName() + ")"); mProviders.add(provider); - if( provider instanceof BaseAuditHandler) { + if (provider instanceof BaseAuditHandler) { BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider; baseAuditHander.setParentPath(getName()); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java index 4278731..2361bbf 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java @@ -39,7 +39,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { Thread consumerThread = null; static int threadCount = 0; - static final String DEFAULT_NAME = "summary"; + static final String DEFAULT_NAME = "batch"; public AuditBatchQueue(AuditHandler consumer) { super(consumer); @@ -209,7 +209,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { long lastDispatchTime = System.currentTimeMillis(); boolean isDestActive = true; while (true) { - logStatusIfRequired(true); + logStatusIfRequired(); // Time to next dispatch long nextDispatchDuration = lastDispatchTime @@ -347,6 +347,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable { } catch (Throwable t) { logger.error("Error while calling stop on consumer.", t); } + logStatus(); logger.info("Exiting consumerThread.run() method. name=" + getName()); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java index 50a74b5..057f192 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java @@ -33,8 +33,9 @@ public abstract class AuditQueue extends BaseAuditHandler { public static final int AUDIT_MAX_QUEUE_SIZE_DEFAULT = 1024 * 1024; public static final int AUDIT_BATCH_INTERVAL_DEFAULT_MS = 1000; public static final int AUDIT_BATCH_SIZE_DEFAULT = 1000; - - //This is the max time the consumer thread will wait before exiting the loop + + // This is the max time the consumer thread will wait before exiting the + // loop public static final int AUDIT_CONSUMER_THREAD_WAIT_MS = 5000; private int maxQueueSize = AUDIT_MAX_QUEUE_SIZE_DEFAULT; @@ -60,14 +61,18 @@ public abstract class AuditQueue extends BaseAuditHandler { protected int fileSpoolMaxWaitTime = 5 * 60 * 1000; // Default 5 minutes protected int fileSpoolDrainThresholdPercent = 80; - //This is set when the first time stop is called. + // This is set when the first time stop is called. protected long stopTime = 0; - + /** * @param consumer */ public AuditQueue(AuditHandler consumer) { this.consumer = consumer; + if (consumer instanceof BaseAuditHandler) { + BaseAuditHandler baseAuditHander = (BaseAuditHandler) consumer; + baseAuditHander.setParentPath(getName()); + } } @Override @@ -111,9 +116,18 @@ public abstract class AuditQueue extends BaseAuditHandler { } @Override + public void setParentPath(String parentPath) { + super.setParentPath(parentPath); + if (consumer != null && consumer instanceof BaseAuditHandler) { + BaseAuditHandler base = (BaseAuditHandler) consumer; + base.setParentPath(getName()); + } + } + + @Override public void setName(String name) { super.setName(name); - if( consumer != null && consumer instanceof BaseAuditHandler) { + if (consumer != null && consumer instanceof BaseAuditHandler) { BaseAuditHandler base = (BaseAuditHandler) consumer; base.setParentPath(getName()); } @@ -126,7 +140,7 @@ public abstract class AuditQueue extends BaseAuditHandler { public boolean isDrainMaxTimeElapsed() { return (stopTime - System.currentTimeMillis()) > AUDIT_CONSUMER_THREAD_WAIT_MS; } - + public boolean isDrain() { return isDrain; }