ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject incubator-ranger git commit: RANGER-397 Incorporate review comments
Date Tue, 02 Jun 2015 04:45:23 GMT
Repository: incubator-ranger
Updated Branches:
  refs/heads/master c52641711 -> 8469e643d


RANGER-397 Incorporate review comments

Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/8469e643
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/8469e643
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/8469e643

Branch: refs/heads/master
Commit: 8469e643da3a81312aff58dc74673ed48e440f95
Parents: c526417
Author: Don Bosco Durai <bosco@apache.org>
Authored: Mon Jun 1 18:02:42 2015 -0700
Committer: Don Bosco Durai <bosco@apache.org>
Committed: Mon Jun 1 18:02:42 2015 -0700

----------------------------------------------------------------------
 .../audit/destination/DBAuditDestination.java   |  3 ++-
 .../audit/destination/FileAuditDestination.java |  3 ++-
 .../audit/destination/HDFSAuditDestination.java |  7 ++++--
 .../destination/Log4JAuditDestination.java      | 15 ++++++++---
 .../audit/destination/SolrAuditDestination.java |  8 +++++-
 .../ranger/audit/provider/BaseAuditHandler.java | 14 +++++++++--
 .../audit/provider/MultiDestAuditProvider.java  | 21 +++++++++++++---
 .../ranger/audit/queue/AuditBatchQueue.java     |  5 ++--
 .../apache/ranger/audit/queue/AuditQueue.java   | 26 +++++++++++++++-----
 9 files changed, 79 insertions(+), 23 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
index 6825637..3d31c06 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
@@ -78,7 +78,7 @@ public class DBAuditDestination extends AuditDestination {
 	@Override
 	public boolean log(Collection<AuditEventBase> events) {
 		boolean retValue = false;
-		logStatusIfRequired(true);
+		logStatusIfRequired();
 		addTotalCount(events.size());
 		
 		if (beginTransaction()) {
@@ -210,6 +210,7 @@ public class DBAuditDestination extends AuditDestination {
 			entityManagerFactory = null;
 			daoManager = null;
 		}
+		logStatus();
 	}
 
 	private EntityManager getEntityManager() {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
index 77c8f61..c6cd8b2 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
@@ -106,7 +106,7 @@ public class FileAuditDestination extends AuditDestination {
 
 	@Override
 	synchronized public boolean logJSON(Collection<String> events) {
-		logStatusIfRequired(true);
+		logStatusIfRequired();
 		addTotalCount(events.size());
 
 		if (isStopped) {
@@ -183,6 +183,7 @@ public class FileAuditDestination extends AuditDestination {
 			}
 			logWriter = null;
 		}
+		logStatus();
 	}
 
 	// Helper methods in this class

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
index b63fa40..e79e42d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
@@ -105,7 +105,7 @@ public class HDFSAuditDestination extends AuditDestination {
 
 	@Override
 	synchronized public boolean logJSON(Collection<String> events) {
-		logStatusIfRequired(true);
+		logStatusIfRequired();
 		addTotalCount(events.size());
 
 		if (!initDone) {
@@ -142,8 +142,9 @@ public class HDFSAuditDestination extends AuditDestination {
 	@Override
 	public boolean log(Collection<AuditEventBase> events) {
 		if (isStopped) {
-			logStatusIfRequired(true);
+			logStatusIfRequired();
 			addTotalCount(events.size());
+			addDeferredCount(events.size());
 			logError("log() called after stop was requested. name=" + getName());
 			return false;
 		}
@@ -153,6 +154,7 @@ public class HDFSAuditDestination extends AuditDestination {
 				jsonList.add(MiscUtil.stringify(event));
 			} catch (Throwable t) {
 				logger.error("Error converting to JSON. event=" + event);
+				addTotalCount(1);				
 				addFailedCount(1);
 				logFailedEvent(event);
 			}
@@ -185,6 +187,7 @@ public class HDFSAuditDestination extends AuditDestination {
 			}
 			logWriter = null;
 		}
+		logStatus();
 	}
 
 	// Helper methods in this class

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
index b57cd86..9521a4a 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
@@ -59,10 +59,17 @@ public class Log4JAuditDestination extends AuditDestination {
 				+ ", loggerName=" + loggerName);
 	}
 
+	
+	@Override
+	public void stop() {
+		super.stop();
+		logStatus();
+	}
+
 	@Override
 	public boolean log(AuditEventBase event) {
 		if (!auditLogger.isInfoEnabled()) {
-			logStatusIfRequired(true);
+			logStatusIfRequired();
 			addTotalCount(1);
 			return true;
 		}
@@ -77,7 +84,7 @@ public class Log4JAuditDestination extends AuditDestination {
 	@Override
 	public boolean log(Collection<AuditEventBase> events) {
 		if (!auditLogger.isInfoEnabled()) {
-			logStatusIfRequired(true);
+			logStatusIfRequired();
 			addTotalCount(events.size());
 			return true;
 		}
@@ -90,7 +97,7 @@ public class Log4JAuditDestination extends AuditDestination {
 
 	@Override
 	public boolean logJSON(String event) {
-		logStatusIfRequired(true);
+		logStatusIfRequired();
 		addTotalCount(1);
 		if (!auditLogger.isInfoEnabled()) {
 			return true;
@@ -106,7 +113,7 @@ public class Log4JAuditDestination extends AuditDestination {
 	@Override
 	public boolean logJSON(Collection<String> events) {
 		if (!auditLogger.isInfoEnabled()) {
-			logStatusIfRequired(true);
+			logStatusIfRequired();
 			addTotalCount(events.size());
 			return true;
 		}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
index b43e573..213e4b2 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
@@ -55,6 +55,12 @@ public class SolrAuditDestination extends AuditDestination {
 		super.init(props, propPrefix);
 		connect();
 	}
+	
+	@Override
+	public void stop() {
+		super.stop();
+		logStatus();
+	}
 
 	synchronized void connect() {
 		if (solrClient == null) {
@@ -103,7 +109,7 @@ public class SolrAuditDestination extends AuditDestination {
 	@Override
 	public boolean log(Collection<AuditEventBase> events) {
 		try {
-			logStatusIfRequired(true);
+			logStatusIfRequired();
 			addTotalCount(events.size());
 			
 			if (solrClient == null) {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
index ef85743..3859a7e 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
@@ -234,9 +234,17 @@ public abstract class BaseAuditHandler implements AuditHandler {
 		return lastDeferredCount;
 	}
 
-	public void logStatusIfRequired(boolean ifRequired) {
+	public void logStatusIfRequired() {
 		long currTime = System.currentTimeMillis();
-		if (!ifRequired || (currTime - lastStatusLogTime) > statusLogIntervalMS) {
+		if ((currTime - lastStatusLogTime) > statusLogIntervalMS) {
+			logStatus();
+		}
+	}
+
+	public void logStatus() {
+		try {
+			long currTime = System.currentTimeMillis();
+
 			long diffTime = currTime - lastStatusLogTime;
 			lastStatusLogTime = currTime;
 
@@ -278,6 +286,8 @@ public abstract class BaseAuditHandler implements AuditHandler {
 					+ (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
 							: "");
 			LOG.info(msg);
+		} catch (Throwable t) {
+			LOG.error("Error while printing stats. auditProvider=" + getName());
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
index 9a5ff13..26108ca 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
@@ -33,13 +33,16 @@ public class MultiDestAuditProvider extends BaseAuditHandler {
 
 	protected List<AuditHandler> mProviders = new ArrayList<AuditHandler>();
 	static final String DEFAULT_NAME = "multi_dest";
-	
+
 	public MultiDestAuditProvider() {
 		LOG.info("MultiDestAuditProvider: creating..");
 		setName(DEFAULT_NAME);
 	}
 
 	public MultiDestAuditProvider(AuditHandler provider) {
+		LOG.info("MultiDestAuditProvider(): provider="
+				+ (provider == null ? null : provider.getName()));
+		setName(DEFAULT_NAME);
 		addAuditProvider(provider);
 	}
 
@@ -59,12 +62,22 @@ public class MultiDestAuditProvider extends BaseAuditHandler {
 		}
 	}
 
-	
+	@Override
+	public void setParentPath(String parentPath) {
+		super.setParentPath(parentPath);
+		for (AuditHandler provider : mProviders) {
+			if (provider instanceof BaseAuditHandler) {
+				BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider;
+				baseAuditHander.setParentPath(getName());
+			}
+		}
+	}
+
 	@Override
 	public void setName(String name) {
 		super.setName(name);
 		for (AuditHandler provider : mProviders) {
-			if( provider instanceof BaseAuditHandler) {
+			if (provider instanceof BaseAuditHandler) {
 				BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider;
 				baseAuditHander.setParentPath(getName());
 			}
@@ -77,7 +90,7 @@ public class MultiDestAuditProvider extends BaseAuditHandler {
 					+ provider.getClass().getCanonicalName() + ")");
 
 			mProviders.add(provider);
-			if( provider instanceof BaseAuditHandler) {
+			if (provider instanceof BaseAuditHandler) {
 				BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider;
 				baseAuditHander.setParentPath(getName());
 			}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
index 4278731..2361bbf 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
@@ -39,7 +39,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 
 	Thread consumerThread = null;
 	static int threadCount = 0;
-	static final String DEFAULT_NAME = "summary";
+	static final String DEFAULT_NAME = "batch";
 
 	public AuditBatchQueue(AuditHandler consumer) {
 		super(consumer);
@@ -209,7 +209,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 		long lastDispatchTime = System.currentTimeMillis();
 		boolean isDestActive = true;
 		while (true) {
-			logStatusIfRequired(true);
+			logStatusIfRequired();
 
 			// Time to next dispatch
 			long nextDispatchDuration = lastDispatchTime
@@ -347,6 +347,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 		} catch (Throwable t) {
 			logger.error("Error while calling stop on consumer.", t);
 		}
+		logStatus();
 		logger.info("Exiting consumerThread.run() method. name=" + getName());
 	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/8469e643/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
index 50a74b5..057f192 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
@@ -33,8 +33,9 @@ public abstract class AuditQueue extends BaseAuditHandler {
 	public static final int AUDIT_MAX_QUEUE_SIZE_DEFAULT = 1024 * 1024;
 	public static final int AUDIT_BATCH_INTERVAL_DEFAULT_MS = 1000;
 	public static final int AUDIT_BATCH_SIZE_DEFAULT = 1000;
-	
-	//This is the max time the consumer thread will wait before exiting the loop 
+
+	// This is the max time the consumer thread will wait before exiting the
+	// loop
 	public static final int AUDIT_CONSUMER_THREAD_WAIT_MS = 5000;
 
 	private int maxQueueSize = AUDIT_MAX_QUEUE_SIZE_DEFAULT;
@@ -60,14 +61,18 @@ public abstract class AuditQueue extends BaseAuditHandler {
 	protected int fileSpoolMaxWaitTime = 5 * 60 * 1000; // Default 5 minutes
 	protected int fileSpoolDrainThresholdPercent = 80;
 
-	//This is set when the first time stop is called.
+	// This is set when the first time stop is called.
 	protected long stopTime = 0;
-	
+
 	/**
 	 * @param consumer
 	 */
 	public AuditQueue(AuditHandler consumer) {
 		this.consumer = consumer;
+		if (consumer instanceof BaseAuditHandler) {
+			BaseAuditHandler baseAuditHander = (BaseAuditHandler) consumer;
+			baseAuditHander.setParentPath(getName());
+		}
 	}
 
 	@Override
@@ -111,9 +116,18 @@ public abstract class AuditQueue extends BaseAuditHandler {
 	}
 
 	@Override
+	public void setParentPath(String parentPath) {
+		super.setParentPath(parentPath);
+		if (consumer != null && consumer instanceof BaseAuditHandler) {
+			BaseAuditHandler base = (BaseAuditHandler) consumer;
+			base.setParentPath(getName());
+		}
+	}
+
+	@Override
 	public void setName(String name) {
 		super.setName(name);
-		if( consumer != null && consumer instanceof BaseAuditHandler) {
+		if (consumer != null && consumer instanceof BaseAuditHandler) {
 			BaseAuditHandler base = (BaseAuditHandler) consumer;
 			base.setParentPath(getName());
 		}
@@ -126,7 +140,7 @@ public abstract class AuditQueue extends BaseAuditHandler {
 	public boolean isDrainMaxTimeElapsed() {
 		return (stopTime - System.currentTimeMillis()) > AUDIT_CONSUMER_THREAD_WAIT_MS;
 	}
-	
+
 	public boolean isDrain() {
 		return isDrain;
 	}


Mime
View raw message