ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From sneet...@apache.org
Subject [1/3] incubator-ranger git commit: RANGER-397 Port log4j audit provider to V3 implementation
Date Mon, 01 Jun 2015 15:14:27 GMT
Repository: incubator-ranger
Updated Branches:
  refs/heads/ranger-0.5 c510b449d -> 681e1ef5c


RANGER-397 Port log4j audit provider to V3 implementation

Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/e92b2d38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/e92b2d38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/e92b2d38

Branch: refs/heads/ranger-0.5
Commit: e92b2d38fa513ae9b92644f9cd72e61818dc41b6
Parents: c510b44
Author: Don Bosco Durai <bosco@apache.org>
Authored: Sun May 31 23:47:26 2015 -0700
Committer: Don Bosco Durai <bosco@apache.org>
Committed: Sun May 31 23:47:26 2015 -0700

----------------------------------------------------------------------
 .../audit/destination/DBAuditDestination.java   |  42 +++---
 .../audit/destination/FileAuditDestination.java |  10 ++
 .../audit/destination/HDFSAuditDestination.java |  11 ++
 .../destination/Log4JAuditDestination.java      | 120 ++++++++++++++++
 .../audit/destination/SolrAuditDestination.java |  11 +-
 .../audit/provider/AuditProviderFactory.java    |   8 +-
 .../ranger/audit/provider/BaseAuditHandler.java | 138 ++++++++++++++++++-
 .../audit/provider/Log4jAuditProvider.java      |   2 +-
 .../audit/provider/MultiDestAuditProvider.java  |  20 ++-
 .../ranger/audit/queue/AuditBatchQueue.java     |   9 ++
 .../ranger/audit/queue/AuditFileSpool.java      |   4 +
 .../apache/ranger/audit/queue/AuditQueue.java   |  30 ++++
 .../admin/client/RangerAdminRESTClient.java     |   7 +-
 .../plugin/audit/RangerDefaultAuditHandler.java |   5 +-
 14 files changed, 384 insertions(+), 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
index 8cece4e..6825637 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
@@ -78,25 +78,32 @@ public class DBAuditDestination extends AuditDestination {
 	@Override
 	public boolean log(Collection<AuditEventBase> events) {
 		boolean retValue = false;
-
-		if (!beginTransaction()) {
-			return false;
-		}
-		boolean isFailed = false;
-		for (AuditEventBase event : events) {
-			try {
-				event.persist(daoManager);
-			} catch (Throwable t) {
-				logger.error("Error persisting data. event=" + event, t);
-				isFailed = true;
-				break;
+		logStatusIfRequired(true);
+		addTotalCount(events.size());
+		
+		if (beginTransaction()) {
+			boolean isFailed = false;
+			for (AuditEventBase event : events) {
+				try {
+					event.persist(daoManager);
+				} catch (Throwable t) {
+					logger.error("Error persisting data. event=" + event, t);
+					isFailed = true;
+					break;
+				}
+			}
+			if (isFailed) {
+				retValue = false;
+				rollbackTransaction();
+			} else {
+				retValue = commitTransaction();
 			}
 		}
-		if (isFailed) {
-			retValue = false;
-			rollbackTransaction();
+		
+		if (retValue) {
+			addSuccessCount(events.size());
 		} else {
-			retValue = commitTransaction();
+			addDeferredCount(events.size());
 		}
 		return retValue;
 	}
@@ -179,6 +186,9 @@ public class DBAuditDestination extends AuditDestination {
 			if (daoManager.getEntityManager() == null) {
 				logger.error("Error connecting audit database. EntityManager is null. dbURL="
 						+ jdbcURL + ", dbUser=" + dbUser);
+			} else {
+				logger.info("Connected to audit database. dbURL=" + jdbcURL
+						+ ", dbUser=" + dbUser);
 			}
 
 		} catch (Throwable t) {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
index a132cdf..854592d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/FileAuditDestination.java
@@ -106,8 +106,11 @@ public class FileAuditDestination extends AuditDestination {
 
 	@Override
 	synchronized public boolean logJSON(Collection<String> events) {
+		logStatusIfRequired(true);
 		if (isStopped) {
 			logError("log() called after stop was requested. name=" + getName());
+			addTotalCount(events.size());
+			addDeferredCount(events.size());
 			return false;
 		}
 
@@ -118,9 +121,11 @@ public class FileAuditDestination extends AuditDestination {
 			}
 			out.flush();
 		} catch (Throwable t) {
+			addDeferredCount(events.size());
 			logError("Error writing to log file.", t);
 			return false;
 		}
+		addSuccessCount(events.size());
 		return true;
 	}
 
@@ -133,6 +138,8 @@ public class FileAuditDestination extends AuditDestination {
 	@Override
 	public boolean log(Collection<AuditEventBase> events) {
 		if (isStopped) {
+			addTotalCount(events.size());
+			addDeferredCount(events.size());
 			logError("log() called after stop was requested. name=" + getName());
 			return false;
 		}
@@ -141,6 +148,9 @@ public class FileAuditDestination extends AuditDestination {
 			try {
 				jsonList.add(MiscUtil.stringify(event));
 			} catch (Throwable t) {
+				addTotalCount(1);
+				addFailedCount(1);
+				logFailedEvent(event);
 				logger.error("Error converting to JSON. event=" + event);
 			}
 		}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
index 67382a9..b63fa40 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/HDFSAuditDestination.java
@@ -105,10 +105,15 @@ public class HDFSAuditDestination extends AuditDestination {
 
 	@Override
 	synchronized public boolean logJSON(Collection<String> events) {
+		logStatusIfRequired(true);
+		addTotalCount(events.size());
+
 		if (!initDone) {
+			addDeferredCount(events.size());
 			return false;
 		}
 		if (isStopped) {
+			addDeferredCount(events.size());
 			logError("log() called after stop was requested. name=" + getName());
 			return false;
 		}
@@ -120,9 +125,11 @@ public class HDFSAuditDestination extends AuditDestination {
 			}
 			out.flush();
 		} catch (Throwable t) {
+			addDeferredCount(events.size());
 			logError("Error writing to log file.", t);
 			return false;
 		}
+		addSuccessCount(events.size());
 		return true;
 	}
 
@@ -135,6 +142,8 @@ public class HDFSAuditDestination extends AuditDestination {
 	@Override
 	public boolean log(Collection<AuditEventBase> events) {
 		if (isStopped) {
+			logStatusIfRequired(true);
+			addTotalCount(events.size());
 			logError("log() called after stop was requested. name=" + getName());
 			return false;
 		}
@@ -144,6 +153,8 @@ public class HDFSAuditDestination extends AuditDestination {
 				jsonList.add(MiscUtil.stringify(event));
 			} catch (Throwable t) {
 				logger.error("Error converting to JSON. event=" + event);
+				addFailedCount(1);
+				logFailedEvent(event);
 			}
 		}
 		return logJSON(jsonList);

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
new file mode 100644
index 0000000..b57cd86
--- /dev/null
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/Log4JAuditDestination.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.destination;
+
+import java.util.Collection;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.provider.MiscUtil;
+
+public class Log4JAuditDestination extends AuditDestination {
+	private static final Log logger = LogFactory
+			.getLog(FileAuditDestination.class);
+
+	private static Log auditLogger = null;
+
+	public static final String PROP_LOG4J_LOGGER = "logger";
+	public static final String DEFAULT_LOGGER_PREFIX = "ranger.audit";
+	private String loggerName = null;
+
+	public Log4JAuditDestination() {
+		logger.info("Log4JAuditDestination() called.");
+
+	}
+
+	@Override
+	public void init(Properties prop, String propPrefix) {
+		super.init(prop, propPrefix);
+		loggerName = MiscUtil.getStringProperty(props, propPrefix + "."
+				+ PROP_LOG4J_LOGGER);
+		if (loggerName == null || loggerName.isEmpty()) {
+			loggerName = DEFAULT_LOGGER_PREFIX + "." + getName();
+			logger.info("Logger property " + propPrefix + "."
+					+ PROP_LOG4J_LOGGER + " was not set. Constructing default="
+					+ loggerName);
+		}
+		logger.info("Logger name for " + getName() + " is " + loggerName);
+		auditLogger = LogFactory.getLog(loggerName);
+		logger.info("Done initializing logger for audit. name=" + getName()
+				+ ", loggerName=" + loggerName);
+	}
+
+	@Override
+	public boolean log(AuditEventBase event) {
+		if (!auditLogger.isInfoEnabled()) {
+			logStatusIfRequired(true);
+			addTotalCount(1);
+			return true;
+		}
+
+		if (event != null) {
+			String eventStr = MiscUtil.stringify(event);
+			logJSON(eventStr);
+		}
+		return true;
+	}
+
+	@Override
+	public boolean log(Collection<AuditEventBase> events) {
+		if (!auditLogger.isInfoEnabled()) {
+			logStatusIfRequired(true);
+			addTotalCount(events.size());
+			return true;
+		}
+
+		for (AuditEventBase event : events) {
+			log(event);
+		}
+		return true;
+	}
+
+	@Override
+	public boolean logJSON(String event) {
+		logStatusIfRequired(true);
+		addTotalCount(1);
+		if (!auditLogger.isInfoEnabled()) {
+			return true;
+		}
+
+		if (event != null) {
+			auditLogger.info(event);
+			addSuccessCount(1);
+		}
+		return true;
+	}
+
+	@Override
+	public boolean logJSON(Collection<String> events) {
+		if (!auditLogger.isInfoEnabled()) {
+			logStatusIfRequired(true);
+			addTotalCount(events.size());
+			return true;
+		}
+
+		for (String event : events) {
+			logJSON(event);
+		}
+		return false;
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
index ac522cd..b43e573 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/SolrAuditDestination.java
@@ -103,10 +103,14 @@ public class SolrAuditDestination extends AuditDestination {
 	@Override
 	public boolean log(Collection<AuditEventBase> events) {
 		try {
+			logStatusIfRequired(true);
+			addTotalCount(events.size());
+			
 			if (solrClient == null) {
 				connect();
 				if (solrClient == null) {
-					// Solr is still not initialized. So need to throw error
+					// Solr is still not initialized. So need return error
+					addDeferredCount(events.size());
 					return false;
 				}
 			}
@@ -121,12 +125,17 @@ public class SolrAuditDestination extends AuditDestination {
 			try {
 				UpdateResponse response = solrClient.add(docs);
 				if (response.getStatus() != 0) {
+					addFailedCount(events.size());
 					logFailedEvent(events, response.toString());
+				} else {
+					addSuccessCount(events.size());
 				}
 			} catch (SolrException ex) {
+				addFailedCount(events.size());
 				logFailedEvent(events, ex);
 			}
 		} catch (Throwable t) {
+			addDeferredCount(events.size());
 			logError("Error sending message to Solr", t);
 			return false;
 		}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index c3a05ce..f02ba62 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -27,6 +27,7 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.audit.destination.DBAuditDestination;
 import org.apache.ranger.audit.destination.FileAuditDestination;
 import org.apache.ranger.audit.destination.HDFSAuditDestination;
+import org.apache.ranger.audit.destination.Log4JAuditDestination;
 import org.apache.ranger.audit.destination.SolrAuditDestination;
 import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider;
 import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider;
@@ -62,6 +63,7 @@ public class AuditProviderFactory {
 	private static AuditProviderFactory sFactory;
 
 	private AuditHandler mProvider = null;
+	private String componentAppType = "";
 	private boolean mInitDone = false;
 
 	private AuditProviderFactory() {
@@ -105,7 +107,7 @@ public class AuditProviderFactory {
 			// return;
 		}
 		mInitDone = true;
-
+		componentAppType = appType;
 		MiscUtil.setApplicationType(appType);
 
 		boolean isEnabled = MiscUtil.getBooleanProperty(props,
@@ -240,7 +242,7 @@ public class AuditProviderFactory {
 			AuditAsyncQueue asyncQueue = new AuditAsyncQueue(consumer);
 			propPrefix = BaseAuditHandler.PROP_DEFAULT_PREFIX + "." + "async";
 			asyncQueue.init(props, propPrefix);
-
+			asyncQueue.setParentPath(componentAppType);
 			mProvider = asyncQueue;
 			LOG.info("Starting audit queue " + mProvider.getName());
 			mProvider.start();
@@ -418,7 +420,7 @@ public class AuditProviderFactory {
 			} else if (providerName.equals("db")) {
 				provider = new DBAuditDestination();
 			} else if (providerName.equals("log4j")) {
-				provider = new Log4jAuditProvider();
+				provider = new Log4JAuditDestination();
 			} else if (providerName.equals("batch")) {
 				provider = new AuditBatchQueue(consumer);
 			} else if (providerName.equals("async")) {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
index 09335c7..ef85743 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditHandler.java
@@ -35,8 +35,7 @@ public abstract class BaseAuditHandler implements AuditHandler {
 	private static final Log LOG = LogFactory.getLog(BaseAuditHandler.class);
 
 	static final String AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP = "xasecure.audit.log.failure.report.min.interval.ms";
-	protected static final String AUDIT_DB_CREDENTIAL_PROVIDER_FILE   = "xasecure.audit.credential.provider.file";
-
+	protected static final String AUDIT_DB_CREDENTIAL_PROVIDER_FILE = "xasecure.audit.credential.provider.file";
 
 	private int mLogFailureReportMinIntervalInMs = 60 * 1000;
 
@@ -52,6 +51,7 @@ public abstract class BaseAuditHandler implements AuditHandler {
 	protected String propPrefix = PROP_DEFAULT_PREFIX;
 
 	protected String providerName = null;
+	protected String parentPath = null;
 
 	protected int failedRetryTimes = 3;
 	protected int failedRetrySleep = 3 * 1000;
@@ -59,6 +59,21 @@ public abstract class BaseAuditHandler implements AuditHandler {
 	int errorLogIntervalMS = 30 * 1000; // Every 30 seconds
 	long lastErrorLogMS = 0;
 
+	long totalCount = 0;
+	long totalSuccessCount = 0;
+	long totalFailedCount = 0;
+	long totalStashedCount = 0;
+	long totalDeferredCount = 0;
+
+	long lastIntervalCount = 0;
+	long lastIntervalSuccessCount = 0;
+	long lastIntervalFailedCount = 0;
+	long lastStashedCount = 0;
+	long lastDeferredCount = 0;
+
+	long lastStatusLogTime = System.currentTimeMillis();
+	long statusLogIntervalMS = 1 * 60 * 1000;
+
 	protected Properties props = null;
 
 	@Override
@@ -81,14 +96,14 @@ public abstract class BaseAuditHandler implements AuditHandler {
 		String name = MiscUtil.getStringProperty(props, basePropertyName + "."
 				+ PROP_NAME);
 		if (name != null && !name.isEmpty()) {
-			providerName = name;
+			setName(name);
 		}
 		if (providerName == null) {
-			providerName = finalToken;
+			setName(finalToken);
 			LOG.info("Using providerName from property prefix. providerName="
-					+ providerName);
+					+ getName());
 		}
-		LOG.info("providerName=" + providerName);
+		LOG.info("providerName=" + getName());
 
 		try {
 			new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create();
@@ -146,15 +161,126 @@ public abstract class BaseAuditHandler implements AuditHandler {
 		return log(eventList);
 	}
 
+	public String getParentPath() {
+		return parentPath;
+	}
+
+	public void setParentPath(String parentPath) {
+		this.parentPath = parentPath;
+	}
+
 	public void setName(String name) {
 		providerName = name;
 	}
 
 	@Override
 	public String getName() {
+		if (parentPath != null) {
+			return parentPath + "." + providerName;
+		}
 		return providerName;
 	}
 
+	public long addTotalCount(int count) {
+		totalCount += count;
+		return totalCount;
+	}
+
+	public long addSuccessCount(int count) {
+		totalSuccessCount += count;
+		return totalSuccessCount;
+	}
+
+	public long addFailedCount(int count) {
+		totalFailedCount += count;
+		return totalFailedCount;
+	}
+
+	public long addStashedCount(int count) {
+		totalStashedCount += count;
+		return totalStashedCount;
+	}
+
+	public long addDeferredCount(int count) {
+		totalDeferredCount += count;
+		return totalDeferredCount;
+	}
+
+	public long getTotalCount() {
+		return totalCount;
+	}
+
+	public long getTotalSuccessCount() {
+		return totalSuccessCount;
+	}
+
+	public long getTotalFailedCount() {
+		return totalFailedCount;
+	}
+
+	public long getTotalStashedCount() {
+		return totalStashedCount;
+	}
+
+	public long getLastStashedCount() {
+		return lastStashedCount;
+	}
+
+	public long getTotalDeferredCount() {
+		return totalDeferredCount;
+	}
+
+	public long getLastDeferredCount() {
+		return lastDeferredCount;
+	}
+
+	public void logStatusIfRequired(boolean ifRequired) {
+		long currTime = System.currentTimeMillis();
+		if (!ifRequired || (currTime - lastStatusLogTime) > statusLogIntervalMS) {
+			long diffTime = currTime - lastStatusLogTime;
+			lastStatusLogTime = currTime;
+
+			long diffCount = totalCount - lastIntervalCount;
+			if (diffCount == 0) {
+				return;
+			}
+			long diffSuccess = totalSuccessCount - lastIntervalSuccessCount;
+			long diffFailed = totalFailedCount - lastIntervalFailedCount;
+			long diffStashed = totalStashedCount - lastStashedCount;
+			long diffDeferred = totalDeferredCount - lastDeferredCount;
+
+			lastIntervalCount = totalCount;
+			lastIntervalSuccessCount = totalSuccessCount;
+			lastIntervalFailedCount = totalFailedCount;
+			lastStashedCount = totalStashedCount;
+			lastDeferredCount = totalDeferredCount;
+
+			String msg = "Audit Status Log: name="
+					+ getName()
+					+ ", interval="
+					+ formatIntervalForLog(diffTime)
+					+ ", events="
+					+ diffCount
+					+ (diffSuccess > 0 ? (", succcessCount=" + diffSuccess)
+							: "")
+					+ (diffFailed > 0 ? (", failedCount=" + diffFailed) : "")
+					+ (diffStashed > 0 ? (", stashedCount=" + diffStashed) : "")
+					+ (diffDeferred > 0 ? (", deferredCount=" + diffDeferred)
+							: "")
+					+ ", totalEvents="
+					+ totalCount
+					+ (totalSuccessCount > 0 ? (", totalSuccessCount=" + totalSuccessCount)
+							: "")
+					+ (totalFailedCount > 0 ? (", totalFailedCount=" + totalFailedCount)
+							: "")
+					+ (totalStashedCount > 0 ? (", totalStashedCount=" + totalStashedCount)
+							: "")
+					+ (totalDeferredCount > 0 ? (", totalDeferredCount=" + totalDeferredCount)
+							: "");
+			LOG.info(msg);
+		}
+	}
+
 	public void logError(String msg) {
 		long currTimeMS = System.currentTimeMillis();
 		if (currTimeMS - lastErrorLogMS > errorLogIntervalMS) {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java
index 0402de2..3beab69 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/Log4jAuditProvider.java
@@ -81,7 +81,7 @@ public class Log4jAuditProvider extends AuditDestination {
 		for (String event : events) {
 			logJSON(event);
 		}
-		return false;
+		return true;
 	}
 
 	@Override

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
index 4c1593a..9a5ff13 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/MultiDestAuditProvider.java
@@ -32,9 +32,11 @@ public class MultiDestAuditProvider extends BaseAuditHandler {
 			.getLog(MultiDestAuditProvider.class);
 
 	protected List<AuditHandler> mProviders = new ArrayList<AuditHandler>();
-
+	static final String DEFAULT_NAME = "multi_dest";
+	
 	public MultiDestAuditProvider() {
 		LOG.info("MultiDestAuditProvider: creating..");
+		setName(DEFAULT_NAME);
 	}
 
 	public MultiDestAuditProvider(AuditHandler provider) {
@@ -57,12 +59,28 @@ public class MultiDestAuditProvider extends BaseAuditHandler {
 		}
 	}
 
+	
+	@Override
+	public void setName(String name) {
+		super.setName(name);
+		for (AuditHandler provider : mProviders) {
+			if( provider instanceof BaseAuditHandler) {
+				BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider;
+				baseAuditHander.setParentPath(getName());
+			}
+		}
+	}
+
 	public void addAuditProvider(AuditHandler provider) {
 		if (provider != null) {
 			LOG.info("MultiDestAuditProvider.addAuditProvider(providerType="
 					+ provider.getClass().getCanonicalName() + ")");
 
 			mProviders.add(provider);
+			if( provider instanceof BaseAuditHandler) {
+				BaseAuditHandler baseAuditHander = (BaseAuditHandler) provider;
+				baseAuditHander.setParentPath(getName());
+			}
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
index 80d7853..4278731 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditBatchQueue.java
@@ -39,9 +39,11 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 
 	Thread consumerThread = null;
 	static int threadCount = 0;
+	static final String DEFAULT_NAME = "summary";
 
 	public AuditBatchQueue(AuditHandler consumer) {
 		super(consumer);
+		setName(DEFAULT_NAME);
 	}
 
 	/*
@@ -207,6 +209,8 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 		long lastDispatchTime = System.currentTimeMillis();
 		boolean isDestActive = true;
 		while (true) {
+			logStatusIfRequired(true);
+
 			// Time to next dispatch
 			long nextDispatchDuration = lastDispatchTime
 					- System.currentTimeMillis() + getMaxBatchInterval();
@@ -269,6 +273,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 				logger.error("Caught error during processing request.", t);
 			}
 
+			addTotalCount(localBatchBuffer.size());
 			if (localBatchBuffer.size() > 0 && isToSpool) {
 				// Let spool to the file directly
 				if (isDestActive) {
@@ -279,6 +284,7 @@ public class AuditBatchQueue extends AuditQueue implements Runnable {
 				// Just before stashing
 				lastDispatchTime = System.currentTimeMillis();
 				fileSpooler.stashLogs(localBatchBuffer);
+				addStashedCount(localBatchBuffer.size());
 				localBatchBuffer.clear();
 			} else if (localBatchBuffer.size() > 0
 					&& (isDrain()
@@ -297,12 +303,15 @@ public class AuditBatchQueue extends AuditQueue implements Runnable
{
 						// Transient error. Stash and move on
 						fileSpooler.stashLogs(localBatchBuffer);
 						isDestActive = false;
+						addStashedCount(localBatchBuffer.size());
 					} else {
 						// We need to drop this event
+						addFailedCount(localBatchBuffer.size());
 						logFailedEvent(localBatchBuffer);
 					}
 				} else {
 					isDestActive = true;
+					addSuccessCount(localBatchBuffer.size());
 				}
 				localBatchBuffer.clear();
 			}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
index 1b9a921..9c10c54 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditFileSpool.java
@@ -842,6 +842,10 @@ public class AuditFileSpool implements Runnable {
 					currentConsumerIndexRecord = null;
 					closeFileIfNeeded();
 				}
+			} catch (InterruptedException e) {
+				logger.info(
+						"Caught exception in consumer thread. Shutdown might be in progress",
+						e);
 			} catch (Throwable t) {
 				logger.error("Exception in destination writing thread.", t);
 			}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
index e873459..7681b82 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/queue/AuditQueue.java
@@ -19,10 +19,12 @@
 
 package org.apache.ranger.audit.queue;
 
+import java.util.Collection;
 import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.audit.model.AuditEventBase;
 import org.apache.ranger.audit.provider.AuditHandler;
 import org.apache.ranger.audit.provider.BaseAuditHandler;
 import org.apache.ranger.audit.provider.MiscUtil;
@@ -110,6 +112,34 @@ public abstract class AuditQueue extends BaseAuditHandler {
 
 	}
 
+	
+	@Override
+	public boolean log(Collection<AuditEventBase> events) {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	@Override
+	public void start() {
+		// TODO Auto-generated method stub
+		
+	}
+
+	@Override
+	public void stop() {
+		// TODO Auto-generated method stub
+		
+	}
+
+	@Override
+	public void setName(String name) {
+		super.setName(name);
+		if( consumer != null && consumer instanceof BaseAuditHandler) {
+			BaseAuditHandler base = (BaseAuditHandler) consumer;
+			base.setParentPath(getName());
+		}
+	}
+
 	public AuditHandler getConsumer() {
 		return consumer;
 	}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
index 9d103bb..51082bd 100644
--- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
+++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
@@ -76,7 +76,8 @@ public class RangerAdminRESTClient implements RangerAdminClient {
 			// no change
 		} else {
 			RESTResponse resp = RESTResponse.fromClientResponse(response);
-
+			LOG.error("Error getting policies. request=" + webResource.toString() 
+					+ ", response=" + response.toString() + ", serviceName=" + serviceName);
 			throw new Exception(resp.getMessage());
 		}
 
@@ -106,7 +107,7 @@ public class RangerAdminRESTClient implements RangerAdminClient {
 
 			throw new Exception("HTTP " + response.getStatus());
 		} else if(response == null) {
-			throw new Exception("unknown error");
+			throw new Exception("unknown error during grantAccess. serviceName="  + serviceName);
 		}
 
 		if(LOG.isDebugEnabled()) {
@@ -133,7 +134,7 @@ public class RangerAdminRESTClient implements RangerAdminClient {
 
 			throw new Exception("HTTP " + response.getStatus());
 		} else if(response == null) {
-			throw new Exception("unknown error");
+			throw new Exception("unknown error. revokeAccess(). serviceName=" + serviceName);
 		}
 
 		if(LOG.isDebugEnabled()) {

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/e92b2d38/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
index 0d38224..253a180 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/audit/RangerDefaultAuditHandler.java
@@ -37,7 +37,7 @@ import org.apache.ranger.plugin.policyengine.RangerAccessResultProcessor;
 
 public class RangerDefaultAuditHandler implements RangerAccessResultProcessor {
 	private static final Log LOG = LogFactory.getLog(RangerDefaultAuditHandler.class);
-
+	static long sequenceNumber = 0;
 
 	public RangerDefaultAuditHandler() {
 	}
@@ -176,7 +176,8 @@ public class RangerDefaultAuditHandler implements RangerAccessResultProcessor
{
 
 		if (auditEvent.getEventId() == null || auditEvent.getEventId().isEmpty()) {
 			auditEvent.setEventId(MiscUtil.generateUniqueId());
-		}		
+		}
+		auditEvent.setSeqNum(sequenceNumber++);
 	}
 
 	public void logAuthzAudits(Collection<AuthzAuditEvent> auditEvents) {


Mime
View raw message