ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [1/2] git commit: ARGUS-5: fix to use logged user context to work with HDFS file system (instead of the current user while the dispatcher thread was created).
Date Tue, 23 Sep 2014 02:30:37 GMT
Repository: incubator-argus
Updated Branches:
  refs/heads/master acd088613 -> 81f76076e


ARGUS-5: fix to use logged user context to work with HDFS file system
(instead of the current user while the dispatcher thread was created).

Project: http://git-wip-us.apache.org/repos/asf/incubator-argus/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-argus/commit/40ab8ebc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-argus/tree/40ab8ebc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-argus/diff/40ab8ebc

Branch: refs/heads/master
Commit: 40ab8ebcd69e5000d8fd44a306edd36d2138ec51
Parents: cfec85a
Author: mneethiraj <mneethiraj@hortonworks.com>
Authored: Mon Sep 22 18:52:04 2014 -0700
Committer: mneethiraj <mneethiraj@hortonworks.com>
Committed: Mon Sep 22 18:52:04 2014 -0700

----------------------------------------------------------------------
 .../audit/provider/LocalFileLogBuffer.java      | 94 +++++++++++++-------
 .../com/xasecure/audit/provider/MiscUtil.java   |  2 +-
 .../audit/provider/hdfs/HdfsLogDestination.java | 15 ++--
 3 files changed, 72 insertions(+), 39 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/40ab8ebc/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
index 753c2e4..c6f2d6f 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/LocalFileLogBuffer.java
@@ -31,10 +31,12 @@ import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.UnsupportedEncodingException;
 import java.io.Writer;
+import java.security.PrivilegedAction;
 import java.util.Arrays;
 import java.util.Comparator;
 import java.util.TreeSet;
 
+import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.log4j.helpers.LogLog;
 
 
@@ -155,7 +157,7 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 	public synchronized boolean add(T log) {
 		boolean ret = false;
 
-		rolloverIfNeeded();
+		checkFileStatus();
 
 		Writer writer = mWriter;
 
@@ -183,11 +185,13 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 	private synchronized void openFile() {
 		LogLog.debug("==> LocalFileLogBuffer.openFile()");
 
-		long currentRolloverStartTime = MiscUtil.getCurrentRolloverStartTime(mNextRolloverTime,
(mRolloverIntervalSeconds * 1000));
-
 		closeFile();
 
-		mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, currentRolloverStartTime);
+		mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds
* 1000));
+
+		long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds
* 1000));
+
+		mBufferFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime);
 
 		FileOutputStream ostream = null;
 		try {
@@ -206,8 +210,6 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 
 		if(mWriter != null) {
 			LogLog.debug("LocalFileLogBuffer.openFile(): opened file " + mBufferFilename);
-
-			mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds
* 1000));
 		} else {
 			LogLog.warn("LocalFileLogBuffer.openFile(): failed to open file for write " + mBufferFilename);
 
@@ -250,11 +252,13 @@ public class LocalFileLogBuffer<T> implements LogBuffer<T>
{
 		LogLog.debug("<== LocalFileLogBuffer.rollover()");
 	}
 
-	private void rolloverIfNeeded() {
+	private void checkFileStatus() {
 		long now = System.currentTimeMillis();
 
 		if(now > mNextRolloverTime) {
 			rollover();
+		} else  if(mWriter == null) {
+			openFile();
 		}
 	}
 
@@ -336,9 +340,33 @@ class DestinationDispatcherThread<T> extends Thread {
 
 	@Override
 	public void run() {
+		UserGroupInformation loginUser = null;
+
+		try {
+			loginUser = UserGroupInformation.getLoginUser();
+		} catch (IOException excp) {
+			LogLog.error("DestinationDispatcherThread.run(): failed to get login user details. Audit
files will not be sent to HDFS destination", excp);
+		}
+
+		if(loginUser == null) {
+			LogLog.error("DestinationDispatcherThread.run(): failed to get login user. Audit files
will not be sent to HDFS destination");
+
+			return;
+		}
+
+		loginUser.doAs(new PrivilegedAction<Integer>() {
+			@Override
+			public Integer run() {
+				doRun();
+
+				return 0;
+			}
+		});
+	}
+
+	private void doRun() {
 		init();
-		
-		// destination start() should be from the dispatcher thread
+
 		mDestination.start();
 
 		int pollIntervalInMs = 1000;
@@ -422,7 +450,7 @@ class DestinationDispatcherThread<T> extends Thread {
 
 		return ret;
 	}
-	
+
 	private String getNextStringifiedLog() {
 		String log = null;
 
@@ -430,25 +458,29 @@ class DestinationDispatcherThread<T> extends Thread {
 			try {
 				while(true) {
 					String line = mReader.readLine();
-					
-					if(line == null) {
+
+					if(line == null) { // reached end-of-file
 						break;
-					} else {
-						if(log == null) {
-							log = "";
-						}
+					}
 
-						if(line.endsWith(MiscUtil.ESCAPE_STR)) {
-							line = line.substring(0, line.length() - MiscUtil.ESCAPE_STR.length());
-	
+					if(line.endsWith(MiscUtil.ESCAPE_STR)) {
+						line = line.substring(0, line.length() - MiscUtil.ESCAPE_STR.length());
+
+						if(log == null) {
+							log = line;
+						} else {
 							log += MiscUtil.LINE_SEPARATOR;
 							log += line;
-							
-							continue;
+						}
+
+						continue;
+					} else {
+						if(log == null) {
+							log = line;
 						} else {
 							log += line;
-							break;
 						}
+						break;
 					}
 				}
 			} catch (IOException excp) {
@@ -508,6 +540,7 @@ class DestinationDispatcherThread<T> extends Thread {
 
 					if(! logFile.renameTo(archiveFile)) {
 						// TODO: renameTo() does not work in all cases. in case of failure, copy the file contents
to the destination and delete the file
+						LogLog.warn("archiving failed to move file: " + mCurrentLogfile + " ==> " + archiveFilename);
 					}
 
 					File   archiveDir = new File(archiveDirName);
@@ -522,17 +555,16 @@ class DestinationDispatcherThread<T> extends Thread {
 
 					if(numOfFilesToDelete > 0) {
 						Arrays.sort(files, new Comparator<File>() {
-
-							@Override
-							public int compare(File f1, File f2) {
-								return (int)(f1.lastModified() - f2.lastModified());
-							}
-						});
+												@Override
+												public int compare(File f1, File f2) {
+													return (int)(f1.lastModified() - f2.lastModified());
+												}
+											});
 
 						for(int i = 0; i < numOfFilesToDelete; i++) {
-							LogLog.debug("DELETE: " + files[i].getAbsolutePath());
-
-							files[i].delete();
+							if(! files[i].delete()) {
+								LogLog.warn("archiving failed to delete file: " + files[i].getAbsolutePath());
+							}
 						}
 					}
 				}

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/40ab8ebc/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
index 3a07710..d382ead 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/MiscUtil.java
@@ -136,7 +136,7 @@ public class MiscUtil {
 		}
 	}
 
-	public static long getCurrentRolloverStartTime(long nextRolloverTime, long interval) {
+	public static long getRolloverStartTime(long nextRolloverTime, long interval) {
 		return (nextRolloverTime <= interval) ? System.currentTimeMillis() : nextRolloverTime
- interval;
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-argus/blob/40ab8ebc/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
index defb39f..0b6dd6e 100644
--- a/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
+++ b/agents-audit/src/main/java/com/xasecure/audit/provider/hdfs/HdfsLogDestination.java
@@ -136,7 +136,7 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 	public boolean sendStringified(String log) {
 		boolean ret = false;
 
-		checkDestinationFileStatus();
+		checkFileStatus();
 
 		OutputStreamWriter writer = mWriter;
 
@@ -158,11 +158,13 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 	private void openFile() {
 		LogLog.debug("==> HdfsLogDestination.openFile()");
 
-		long currentRolloverStartTime = MiscUtil.getCurrentRolloverStartTime(mNextRolloverTime,
(mRolloverIntervalSeconds * 1000));
-
 		closeFile();
 
-		mHdfsFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, currentRolloverStartTime);
+		mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds
* 1000));
+
+		long startTime = MiscUtil.getRolloverStartTime(mNextRolloverTime, (mRolloverIntervalSeconds
* 1000));
+
+		mHdfsFilename = MiscUtil.replaceTokens(mDirectory + File.separator + mFile, startTime);
 
 		FSDataOutputStream ostream     = null;
 		FileSystem         fileSystem  = null;
@@ -186,7 +188,7 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 						ostream = fileSystem.append(pathLogfile);
 					} catch(IOException excp) {
 						// append may not be supported by the filesystem. rename existing file and create a
new one
-						String fileSuffix    = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_TIME_START + "yyyyMMdd-HHmm.ss"
+ MiscUtil.TOKEN_TIME_END, currentRolloverStartTime);
+						String fileSuffix    = MiscUtil.replaceTokens("-" + MiscUtil.TOKEN_TIME_START + "yyyyMMdd-HHmm.ss"
+ MiscUtil.TOKEN_TIME_END, startTime);
 						String movedFilename = appendToFilename(mHdfsFilename, fileSuffix);
 						Path   movedFilePath = new Path(movedFilename);
 
@@ -222,7 +224,6 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 		if(mWriter != null) {
 			LogLog.debug("HdfsLogDestination.openFile(): opened file " + mHdfsFilename);
 
-			mNextRolloverTime = MiscUtil.getNextRolloverTime(mNextRolloverTime, (mRolloverIntervalSeconds
* 1000));
 			mLastOpenFailedTime = 0;
 		} else {
 			LogLog.warn("HdfsLogDestination.openFile(): failed to open file for write " + mHdfsFilename);
@@ -265,7 +266,7 @@ public class HdfsLogDestination<T> implements LogDestination<T>
{
 		LogLog.debug("<== HdfsLogDestination.rollover()");
 	}
 
-	private void checkDestinationFileStatus() {
+	private void checkFileStatus() {
 		long now = System.currentTimeMillis();
 
 		if(mWriter == null) {


Mime
View raw message