ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [1/3] git commit: ARGUS-97: updated error handling in audit framework, with the following changes: - Handle errors during audit framework initialization due to failure in retrieving db password from the credential store - HDFS file logger initializatio
Date Fri, 03 Oct 2014 01:56:58 GMT
Repository: incubator-argus
Updated Branches:
  refs/heads/master 83c336de5 -> b83ae91f3


ARGUS-97: updated error handling in audit framework, with the following
changes:
 - Handle errors during audit framework initialization due to failure in
retrieving db password from the credential store
 - HDFS file logger initialization error due to missing configuration in
xasecure-audit.xml 
 - Introduced 'db retry min interval' configuration to control the wait
time before attempting to connect to the database after a failure
 - DbAuditProvider updated to handle exceptions while saving to DB,
instead of propagating to the caller; the provider also prints a warning
log, in log4j, with details of the audit event that failed to make to DB


Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/1927499d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/1927499d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/1927499d

Branch: refs/heads/master
Commit: 1927499d9ce4e5e5e482db272bd4c80138e55c9d
Parents: 91dbc72
Author: mneethiraj <mneethiraj@hortonworks.com>
Authored: Thu Oct 2 16:00:48 2014 -0700
Committer: mneethiraj <mneethiraj@hortonworks.com>
Committed: Thu Oct 2 16:00:48 2014 -0700

----------------------------------------------------------------------
 .../audit/provider/AsyncAuditProvider.java      |   8 +-
 .../audit/provider/AuditProviderFactory.java    |   4 +-
 .../audit/provider/DbAuditProvider.java         | 182 +++++++++++++------
 .../audit/provider/LocalFileLogBuffer.java      |  27 +--
 .../utils/XaSecureCredentialProvider.java       |  55 +++---
 5 files changed, 171 insertions(+), 105 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/1927499d/agents-audit/src/main/java/com/xasecure/audit/provider/AsyncAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/AsyncAuditProvider.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/AsyncAuditProvider.java
index f793553..b8de56d 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/AsyncAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/AsyncAuditProvider.java
@@ -127,8 +127,8 @@ public class AsyncAuditProvider extends MultiDestAuditProvider implements
 	public void run() {
 		LOG.info("==> AsyncAuditProvider.run()");
 
-		try {
-			while (!mStopThread) {
+		while (!mStopThread) {
+			try {
 				AuditEventBase event = dequeueEvent();
 
 				if (event != null) {
@@ -136,8 +136,12 @@ public class AsyncAuditProvider extends MultiDestAuditProvider implements
 				} else {
 					flush();
 				}
+			} catch (Exception excp) {
+				LOG.error("AsyncAuditProvider.run()", excp);
 			}
+		}
 
+		try {
 			flush();
 		} catch (Exception excp) {
 			LOG.error("AsyncAuditProvider.run()", excp);

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/1927499d/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
index 814026d..87508aa 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/AuditProviderFactory.java
@@ -52,6 +52,7 @@ public class AuditProviderFactory {
 	private static final String AUDIT_DB_RESUME_QUEUE_SIZE__PROP    = "xasecure.audit.db.async.resume.queue.size"
;
 	private static final String AUDIT_DB_MAX_FLUSH_INTERVAL_PROP    = "xasecure.audit.db.async.max.flush.interval.ms";
 	private static final String AUDIT_DB_BATCH_SIZE_PROP            = "xasecure.audit.db.batch.size"
;
+	private static final String AUDIT_DB_RETRY_MIN_INTERVAL_PROP    = "xasecure.audit.db.config.retry.min.interval.ms";
 	private static final String AUDIT_JPA_CONFIG_PROP_PREFIX        = "xasecure.audit.jpa.";
 	private static final String AUDIT_DB_CREDENTIAL_PROVIDER_FILE   = "xasecure.audit.credential.provider.file";
 	private static final String AUDIT_DB_CREDENTIAL_PROVIDER_ALIAS	= "auditDBCred";
@@ -144,13 +145,14 @@ public class AuditProviderFactory {
 			LOG.info("AuditProviderFactory: found " + jpaInitProperties.size() + " Audit JPA properties");
 	
 			int dbBatchSize          = getIntProperty(props, AUDIT_DB_BATCH_SIZE_PROP, 1000);
+			int dbRetryMinIntervalMs = getIntProperty(props, AUDIT_DB_RETRY_MIN_INTERVAL_PROP, 15
* 1000);
 			boolean isAuditToDbAsync = getBooleanProperty(props, AUDIT_DB_IS_ASYNC_PROP, false);
 			
 			if(! isAuditToDbAsync) {
 				dbBatchSize = 1; // Batching not supported in sync mode; need to address multiple threads
making audit calls
 			}
 
-			DbAuditProvider dbProvider = new DbAuditProvider(jpaInitProperties, dbBatchSize);
+			DbAuditProvider dbProvider = new DbAuditProvider(jpaInitProperties, dbBatchSize, dbRetryMinIntervalMs);
 			
 			if(isAuditToDbAsync) {
 				AsyncAuditProvider asyncProvider = new AsyncAuditProvider();

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/1927499d/agents-audit/src/main/java/com/xasecure/audit/provider/DbAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/DbAuditProvider.java b/agents-audit/src/main/java/com/xasecure/audit/provider/DbAuditProvider.java
index 3add646..f8dc441 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/DbAuditProvider.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/DbAuditProvider.java
@@ -44,25 +44,43 @@ public class DbAuditProvider implements AuditProvider {
 	private EntityManagerFactory entityManagerFactory;
 	private DaoManager          daoManager;
 	
-	private int                 mCommitBatchSize  = 1;
-	private long                mLastCommitTime   = System.currentTimeMillis();
-	private ArrayList<AuditEventBase> mUncommitted = new ArrayList<AuditEventBase>();
-	private Map<String, String> mDbProperties;
-
-	public DbAuditProvider(Map<String, String> properties, int dbBatchSize) {
+	private int                 mCommitBatchSize      = 1;
+	private int                 mDbRetryMinIntervalMs = 60 * 1000;
+	private long                mLastCommitTime       = System.currentTimeMillis();
+	private ArrayList<AuditEventBase> mUncommitted    = new ArrayList<AuditEventBase>();
+	private Map<String, String> mDbProperties         = null;
+	private long                mLastDbFailedTime     = 0;
+
+	public DbAuditProvider(Map<String, String> properties, int dbBatchSize, int dbRetryMinIntervalMs)
{
 		LOG.info("DbAuditProvider: creating..");
 		
-		mDbProperties    = properties;
-		mCommitBatchSize = dbBatchSize < 1 ? 1 : dbBatchSize;
+		mDbProperties         = properties;
+		mCommitBatchSize      = dbBatchSize < 1 ? 1 : dbBatchSize;
+		mDbRetryMinIntervalMs = dbRetryMinIntervalMs;
 	}
 
 	@Override
 	public void log(AuditEventBase event) {
 		LOG.debug("DbAuditProvider.log()");
 
-		if(preCreate(event)) {
-			event.persist(daoManager);
-			postCreate(event);
+		boolean isSuccess = false;
+
+		try {
+			if(preCreate(event)) {
+				DaoManager daoMgr = daoManager;
+	
+				if(daoMgr != null) {
+					event.persist(daoMgr);
+	
+					isSuccess = postCreate(event);
+				}
+			}
+		} catch(Exception excp) {
+			logDbError("DbAuditProvider.log(): failed", excp);
+		} finally {
+			if(! isSuccess) {
+				logFailedEvent(event);
+			}
 		}
 	}
 
@@ -98,38 +116,54 @@ public class DbAuditProvider implements AuditProvider {
 	@Override
 	public void flush() {
 		if(mUncommitted.size() > 0) {
-			commitTransaction();
+			boolean isSuccess = commitTransaction();
+
+			if(! isSuccess) {
+				for(AuditEventBase evt : mUncommitted) {
+					logFailedEvent(evt);
+				}
+			}
+
+			mUncommitted.clear();
 		}
 	}
 
-	private boolean init() {
+	private synchronized boolean init() {
+		long now = System.currentTimeMillis();
+
+		if((now - mLastDbFailedTime) < mDbRetryMinIntervalMs) {
+			return false;
+		}
+
 		LOG.info("DbAuditProvider: init()");
 
 		try {
 			entityManagerFactory = Persistence.createEntityManagerFactory("xa_server", mDbProperties);
+
+	   	    daoManager = new DaoManager();
+	   	    daoManager.setEntityManagerFactory(entityManagerFactory);
+
+	   	    daoManager.getEntityManager(); // this forces the connection to be made to DB
 		} catch(Exception excp) {
-			LOG.error("DbAuditProvider: DB initalization failed", excp);
+			logDbError("DbAuditProvider: DB initalization failed", excp);
 
-			entityManagerFactory = null;
+			cleanUp();
 
 			return false;
 		}
 
-   	    daoManager = new DaoManager();
-   	    daoManager.setEntityManagerFactory(entityManagerFactory);
-
 		return true;
 	}
 	
-	private void cleanUp() {
+	private synchronized void cleanUp() {
 		LOG.info("DbAuditProvider: cleanUp()");
 
 		try {
-			clearEntityManager();
-
 			if(entityManagerFactory != null && entityManagerFactory.isOpen()) {
 				entityManagerFactory.close();
 			}
+		} catch(Exception excp) {
+			LOG.error("DbAuditProvider.cleanUp(): failed", excp);
 		} finally {
 			entityManagerFactory = null;
 			daoManager    = null;
@@ -143,25 +177,36 @@ public class DbAuditProvider implements AuditProvider {
 	}
 	
 	private EntityManager getEntityManager() {
-		return daoManager != null ? daoManager.getEntityManager() : null;
+		DaoManager daoMgr = daoManager;
+
+		if(daoMgr != null) {
+			try {
+				return daoMgr.getEntityManager();
+			} catch(Exception excp) {
+				logDbError("DbAuditProvider.getEntityManager(): failed", excp);
+
+				cleanUp();
+			}
+		}
+
+		return null;
 	}
 	
 	private void clearEntityManager() {
-		EntityManager em = getEntityManager();
-		
-		if(em == null) {
-			LOG.info("clearEntityManager(): em is null");
-		} else {
-			em.clear();
+		try {
+			EntityManager em = getEntityManager();
+			
+			if(em != null) {
+				em.clear();
+			}
+		} catch(Exception excp) {
+			LOG.warn("DbAuditProvider.clearEntityManager(): failed", excp);
 		}
 	}
 	
 	private EntityTransaction getTransaction() {
 		EntityManager em = getEntityManager();
 
-		if(em == null)
-			LOG.info("getTransaction(): em is null");
-
 		return em != null ? em.getTransaction() : null;
 	}
 	
@@ -179,49 +224,43 @@ public class DbAuditProvider implements AuditProvider {
 		}
 
 		if(trx == null) {
-			LOG.error("beginTransaction(): trx is null");
+			LOG.warn("DbAuditProvider.beginTransaction(): trx is null");
 		}
 		
 		return trx != null;
 	}
 
-	private void commitTransaction() {
-		EntityTransaction trx = getTransaction();
+	private boolean commitTransaction() {
+		boolean           ret = false;
+		EntityTransaction trx = null;
 
 		try {
+			trx = getTransaction();
+
 			if(trx != null && trx.isActive()) {
 				trx.commit();
-			} else {
-				if(trx == null) {
-					LOG.error("commitTransaction(): trx is null. Clearing " + mUncommitted.size() + " uncommitted
logs");
-				}
-				else {
-					LOG.error("commitTransaction(): trx is not active. Clearing " + mUncommitted.size()
+ " uncommitted logs");
-				}
 
-				cleanUp(); // so that next insert will try to init()
+				ret =true;
+			} else {
+				throw new Exception("trx is null or not active");
 			}
 		} catch(Exception excp) {
-			LOG.error("commitTransaction(): error while committing " + mUncommitted.size() + " log(s)",
excp);
-			for(AuditEventBase event : mUncommitted) {
-				LOG.error("failed to log event { " + event.toString() + " }");
-			}
+			logDbError("DbAuditProvider.commitTransaction(): failed", excp);
 
 			cleanUp(); // so that next insert will try to init()
 		} finally {
 			mLastCommitTime = System.currentTimeMillis();
-			mUncommitted.clear();
 
 			clearEntityManager();
 		}
+
+		return ret;
 	}
 	
 	private boolean preCreate(AuditEventBase event) {
 		boolean ret = true;
 
 		if(!isDbConnected()) {
-			LOG.error("DbAuditProvider: not connected to DB. Retrying..");
-
 			ret = init();
 		}
 
@@ -231,18 +270,45 @@ public class DbAuditProvider implements AuditProvider {
 			}
 		}
 		
-		if(!ret) {
-			LOG.error("failed to log event { " + event.toString() + " }");
-		}
-		
 		return ret;
 	}
 	
-	private void postCreate(AuditEventBase event) {
-		mUncommitted.add(event);
+	private boolean postCreate(AuditEventBase event) {
+		boolean ret = true;
 
-		if((mCommitBatchSize == 1) || ((mUncommitted.size() % mCommitBatchSize) == 0)) {
-			flush();
-		}
+		if(mCommitBatchSize <= 1) {
+			ret = commitTransaction();
+		} else {
+			mUncommitted.add(event);
+
+			if((mUncommitted.size() % mCommitBatchSize) == 0) {
+				ret = commitTransaction();
+
+				if(! ret) {
+					for(AuditEventBase evt : mUncommitted) {
+						if(evt != event) {
+							logFailedEvent(evt);
+						}
+					}
+				}
+
+				mUncommitted.clear();
+			}
+ 		}
+ 		return ret;
+	}
+
+	private void logDbError(String msg, Exception excp) {
+		long now = System.currentTimeMillis();
+
+		if((now - mLastDbFailedTime) > mDbRetryMinIntervalMs) {
+			mLastDbFailedTime = now;
+ 		}
+
+		LOG.warn(msg, excp);
 	}
+
+	private void logFailedEvent(AuditEventBase event) {
+		LOG.warn("failed to log audit event: " + MiscUtil.stringify(event) + " }");
+ 	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/1927499d/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index d59d1db..f743cc3 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -426,18 +426,21 @@ class DestinationDispatcherThread<T> extends Thread {
 	private void init() {
 		LogLog.debug("==> DestinationDispatcherThread.init()");
 
-		String dirName   = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory(), 0);
-		File   directory = new File(dirName);
-
-		if(directory.exists() && directory.isDirectory()) {
-			File[] files = directory.listFiles();
-
-			if(files != null) {
-				for(File file : files) {
-					if(file.exists() && file.canRead()) {
-						String filename = file.getAbsolutePath();
-						if(! mFileLogBuffer.isCurrentFilename(filename)) {
-							addLogfile(filename);
+		String dirName = MiscUtil.replaceTokens(mFileLogBuffer.getDirectory(), 0);
+		
+		if(dirName != null) {
+			File directory = new File(dirName);
+		
+			if(directory.exists() && directory.isDirectory()) {
+				File[] files = directory.listFiles();
+		
+				if(files != null) {
+					for(File file : files) {
+						if(file.exists() && file.canRead()) {
+							String filename = file.getAbsolutePath();
+							if(! mFileLogBuffer.isCurrentFilename(filename)) {
+								addLogfile(filename);
+							}
 						}
 					}
 				}

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/1927499d/agents-cred/src/main/java/com/xasecure/authorization/hadoop/utils/XaSecureCredentialProvider.java
----------------------------------------------------------------------
diff --git a/agents-cred/src/main/java/com/xasecure/authorization/hadoop/utils/XaSecureCredentialProvider.java
b/agents-cred/src/main/java/com/xasecure/authorization/hadoop/utils/XaSecureCredentialProvider.java
index 07a6357..ceccb7b 100644
--- a/agents-cred/src/main/java/com/xasecure/authorization/hadoop/utils/XaSecureCredentialProvider.java
+++ b/agents-cred/src/main/java/com/xasecure/authorization/hadoop/utils/XaSecureCredentialProvider.java
@@ -33,9 +33,7 @@ import org.apache.commons.logging.LogFactory;
 public class XaSecureCredentialProvider {
 
   private static Log LOG = LogFactory.getLog(XaSecureCredentialProvider.class);
-  private static Configuration conf = null;
 
-  private static List<CredentialProvider> providers = null;
   private static XaSecureCredentialProvider  me = null;
 
   
@@ -45,49 +43,42 @@ public class XaSecureCredentialProvider {
 			  XaSecureCredentialProvider temp = me;
 			  if ( temp == null){
 				  me = new XaSecureCredentialProvider();
-				  me.init();
 			  }
 		  }
 	  }
 	return me;
   }
   
-  
-  private void init() {
-	  conf  = new Configuration();
-  }
-  
   public char[] getCredentialString(String url, String alias)  {
-  
-   char[] pass = null;
- 
-   conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, url);
-   providers =  getCredentialProviders(); 
+   List<CredentialProvider> providers =  getCredentialProviders(url); 
    
-   CredentialProvider.CredentialEntry credEntry = null;
-   
-   for(  CredentialProvider provider: providers) {
-	   try {
-         credEntry = provider.getCredentialEntry(alias);
-         if (credEntry != null) {
-            pass = credEntry.getCredential();
-         } else {
-        	return pass;
-         }
-        } catch(IOException ie) {
-        	LOG.error("Unable to get the Credential Provider from the Configuration", ie);	

-       }
-    }
-   return pass;
+   if(providers != null) {
+	   for(  CredentialProvider provider: providers) {
+		   try {
+			 CredentialProvider.CredentialEntry credEntry = provider.getCredentialEntry(alias);
+
+	         if (credEntry != null) {
+	            return credEntry.getCredential();
+	         }
+	        } catch(Exception ie) {
+	        	LOG.error("Unable to get the Credential Provider from the Configuration", ie);
 
+	       }
+	   }
+   }
+   return null;
   }
   
-  public  List<CredentialProvider>  getCredentialProviders(){
+  private List<CredentialProvider>  getCredentialProviders(String url){
    try {
-       providers = CredentialProviderFactory.getProviders(conf);   
-      } catch( IOException ie) {
+	   Configuration conf = new Configuration();
+
+	   conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, url);
+
+	   return CredentialProviderFactory.getProviders(conf);   
+      } catch(Exception ie) {
     	  LOG.error("Unable to get the Credential Provider from the Configuration", ie);
       }     
-   return providers;
+   return null;
   }
 
 }
\ No newline at end of file


Mime
View raw message