ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a...@apache.org
Subject incubator-ranger git commit: RANGER-615 Audit to db: Truncate all string values of audit record so that writing of audit does not fail
Date Fri, 21 Aug 2015 17:07:38 GMT
Repository: incubator-ranger
Updated Branches:
  refs/heads/ranger-0.5 a451e028e -> 3b3b8d95e


RANGER-615 Audit to db: Truncate all string values of audit record so that writing of audit
does not fail


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/3b3b8d95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/3b3b8d95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/3b3b8d95

Branch: refs/heads/ranger-0.5
Commit: 3b3b8d95ec42089548a128d26c213f1bc0071508
Parents: a451e02
Author: Alok Lal <alok@apache.org>
Authored: Thu Aug 13 08:48:14 2015 -0700
Committer: Alok Lal <alok@apache.org>
Committed: Fri Aug 21 10:02:38 2015 -0700

----------------------------------------------------------------------
 .../audit/destination/DBAuditDestination.java   |   5 +-
 .../audit/entity/AuthzAuditEventDbObj.java      | 143 +++++++++++++++++--
 .../ranger/audit/model/AuditEventBase.java      |  14 +-
 .../ranger/audit/model/AuthzAuditEvent.java     |  12 +-
 .../ranger/audit/provider/DbAuditProvider.java  |   4 +
 5 files changed, 144 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3b3b8d95/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
index 3d31c06..376e724 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/destination/DBAuditDestination.java
@@ -33,6 +33,7 @@ import javax.persistence.EntityTransaction;
 import javax.persistence.Persistence;
 
 import org.apache.ranger.audit.dao.DaoManager;
+import org.apache.ranger.audit.entity.AuthzAuditEventDbObj;
 import org.apache.ranger.audit.model.AuditEventBase;
 import org.apache.ranger.audit.provider.MiscUtil;
 
@@ -63,9 +64,11 @@ public class DBAuditDestination extends AuditDestination {
 	public void init(Properties props, String propPrefix) {
 		logger.info("init() called");
 		super.init(props, propPrefix);
-
 		// Initial connect
 		connect();
+
+		// initialize the database related classes
+		AuthzAuditEventDbObj.init(props);
 	}
 
 	/*

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3b3b8d95/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
b/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
index 435393e..d52a60a 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/entity/AuthzAuditEventDbObj.java
@@ -21,6 +21,7 @@
 
 import java.io.Serializable;
 import java.util.Date;
+import java.util.Properties;
 
 import javax.persistence.Column;
 import javax.persistence.Entity;
@@ -32,8 +33,11 @@ import javax.persistence.Temporal;
 import javax.persistence.TemporalType;
 import javax.persistence.SequenceGenerator;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.audit.model.EnumRepositoryType;
 import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.MiscUtil;
 
 /**
  * Entity implementation class for Entity: AuthzAuditEventDbObj
@@ -42,8 +46,25 @@ import org.apache.ranger.audit.model.AuthzAuditEvent;
 @Entity
 @Table(name="xa_access_audit")
 public class AuthzAuditEventDbObj implements Serializable {
+
+	private static final Log LOG = LogFactory.getLog(AuthzAuditEventDbObj.class);
+
 	private static final long serialVersionUID = 1L;
 
+	static int MaxValueLengthAccessType = 255;
+	static int MaxValueLengthAclEnforcer = 255;
+	static int MaxValueLengthAgentId = 255;
+	static int MaxValueLengthClientIp = 255;
+	static int MaxValueLengthClientType = 255;
+	static int MaxValueLengthRepoName = 255;
+	static int MaxValueLengthResultReason = 255;
+	static int MaxValueLengthSessionId = 255;
+	static int MaxValueLengthRequestUser = 255;
+	static int MaxValueLengthAction = 2000;
+	static int MaxValueLengthRequestData = 4000;
+	static int MaxValueLengthResourcePath = 4000;
+	static int MaxValueLengthResourceType = 255;
+
 	private long   auditId;
 	private int    repositoryType;
 	private String repositoryName;
@@ -63,6 +84,60 @@ public class AuthzAuditEventDbObj implements Serializable {
 	private String clientIP;
 	private String requestData;
 
+	public static void init(Properties props)
+	{
+		LOG.info("AuthzAuditEventDbObj.init()");
+
+		final String AUDIT_DB_MAX_COLUMN_VALUE = "xasecure.audit.destination.db.max.column.length";
+		MaxValueLengthAccessType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "."
+ "access_type", MaxValueLengthAccessType);
+		logMaxColumnValue("access_type", MaxValueLengthAccessType);
+
+		MaxValueLengthAclEnforcer = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE +
"." + "acl_enforcer", MaxValueLengthAclEnforcer);
+		logMaxColumnValue("acl_enforcer", MaxValueLengthAclEnforcer);
+
+		MaxValueLengthAction = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "." +
"action", MaxValueLengthAction);
+		logMaxColumnValue("action", MaxValueLengthAction);
+
+		MaxValueLengthAgentId = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "."
+ "agent_id", MaxValueLengthAgentId);
+		logMaxColumnValue("agent_id", MaxValueLengthAgentId);
+
+		MaxValueLengthClientIp = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "."
+ "client_id", MaxValueLengthClientIp);
+		logMaxColumnValue("client_id", MaxValueLengthClientIp);
+
+		MaxValueLengthClientType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "."
+ "client_type", MaxValueLengthClientType);
+		logMaxColumnValue("client_type", MaxValueLengthClientType);
+
+		MaxValueLengthRepoName = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "."
+ "repo_name", MaxValueLengthRepoName);
+		logMaxColumnValue("repo_name", MaxValueLengthRepoName);
+
+		MaxValueLengthResultReason = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE +
"." + "result_reason", MaxValueLengthResultReason);
+		logMaxColumnValue("result_reason", MaxValueLengthResultReason);
+
+		MaxValueLengthSessionId = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE + "."
+ "session_id", MaxValueLengthSessionId);
+		logMaxColumnValue("session_id", MaxValueLengthSessionId);
+
+		MaxValueLengthRequestUser = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE +
"." + "request_user", MaxValueLengthRequestUser);
+		logMaxColumnValue("request_user", MaxValueLengthRequestUser);
+
+		MaxValueLengthRequestData = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE +
"." + "request_data", MaxValueLengthRequestData);
+		logMaxColumnValue("request_data", MaxValueLengthRequestData);
+
+		MaxValueLengthResourcePath = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE +
"." + "resource_path", MaxValueLengthResourcePath);
+		logMaxColumnValue("resource_path", MaxValueLengthResourcePath);
+
+		MaxValueLengthResourceType = MiscUtil.getIntProperty(props, AUDIT_DB_MAX_COLUMN_VALUE +
"." + "resource_type", MaxValueLengthResourceType);
+		logMaxColumnValue("resource_type", MaxValueLengthResourceType);
+	}
+
+	public static void logMaxColumnValue(String columnName, int configuredMaxValueLength) {
+		LOG.info("Setting max column value for column[" + columnName + "] to [" + configuredMaxValueLength
+ "].");
+		if (configuredMaxValueLength == 0) {
+			LOG.info("Max length of column[" + columnName + "] was 0! Column will NOT be emitted in
the audit.");
+		} else if (configuredMaxValueLength < 0) {
+			LOG.info("Max length of column[" + columnName + "] was less than 0! Column value will
never be truncated.");
+		}
+	}
+
 
 	public AuthzAuditEventDbObj() {
 		super();
@@ -113,7 +188,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "repo_name")
 	public String getRepositoryName() {
-		return this.repositoryName;
+		return truncate(this.repositoryName, MaxValueLengthRepoName, "repo_name");
 	}
 
 	public void setRepositoryName(String repositoryName) {
@@ -122,7 +197,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "request_user")
 	public String getUser() {
-		return this.user;
+		return truncate(this.user, MaxValueLengthRequestUser, "request_user");
 	}
 
 	public void setUser(String user) {
@@ -141,7 +216,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "access_type")
 	public String getAccessType() {
-		return this.accessType;
+		return truncate(this.accessType, MaxValueLengthAccessType, "access_type");
 	}
 
 	public void setAccessType(String accessType) {
@@ -150,7 +225,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "resource_path")
 	public String getResourcePath() {
-		return this.resourcePath;
+		return truncate(this.resourcePath, MaxValueLengthResourcePath, "resource_path");
 	}
 
 	public void setResourcePath(String resourcePath) {
@@ -159,7 +234,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "resource_type")
 	public String getResourceType() {
-		return this.resourceType;
+		return truncate(this.resourceType, MaxValueLengthResourceType, "resource_type");
 	}
 
 	public void setResourceType(String resourceType) {
@@ -168,7 +243,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "action")
 	public String getAction() {
-		return this.action;
+		return truncate(this.action, MaxValueLengthAction, "action");
 	}
 
 	public void setAction(String action) {
@@ -186,7 +261,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "agent_id")
 	public String getAgentId() {
-		return agentId;
+		return truncate(this.agentId, MaxValueLengthAgentId, "agent_id");
 	}
 
 	public void setAgentId(String agentId) {
@@ -204,7 +279,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "result_reason")
 	public String getResultReason() {
-		return this.resultReason;
+		return truncate(this.resultReason, MaxValueLengthResultReason, "result_reason");
 	}
 
 	public void setResultReason(String resultReason) {
@@ -213,7 +288,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "acl_enforcer")
 	public String getAclEnforcer() {
-		return this.aclEnforcer;
+		return truncate(this.aclEnforcer, MaxValueLengthAclEnforcer, "acl_enforcer");
 	}
 
 	public void setAclEnforcer(String aclEnforcer) {
@@ -222,7 +297,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "session_id")
 	public String getSessionId() {
-		return this.sessionId;
+		return truncate(this.sessionId, MaxValueLengthSessionId, "session_id");
 	}
 
 	public void setSessionId(String sessionId) {
@@ -231,7 +306,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "client_type")
 	public String getClientType() {
-		return this.clientType;
+		return truncate(this.clientType, MaxValueLengthClientType, "client_type");
 	}
 
 	public void setClientType(String clientType) {
@@ -240,7 +315,7 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "client_ip")
 	public String getClientIP() {
-		return this.clientIP;
+		return truncate(this.clientIP, MaxValueLengthClientIp, "client_ip");
 	}
 
 	public void setClientIP(String clientIP) {
@@ -249,10 +324,52 @@ public class AuthzAuditEventDbObj implements Serializable {
 
 	@Column(name = "request_data")
 	public String getRequestData() {
-		return this.requestData;
+		return truncate(this.requestData, MaxValueLengthRequestData, "request_data");
 	}
 
 	public void setRequestData(String requestData) {
 		this.requestData = requestData;
 	}
+	static final String TruncationMarker = "...";
+	static final int TruncationMarkerLength = TruncationMarker.length();
+
+	protected String truncate(String value, int limit, String columnName) {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug(String.format("==> getTrunctedValue(%s, %d, %s)", value, limit, columnName));
+		}
+
+		String result = value;
+		if (value != null) {
+			if (limit < 0) {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug(String.format("Truncation is suppressed for column[%s]: old value [%s], new
value[%s]", columnName, value, result));
+				}
+			} else if (limit == 0) {
+				if (LOG.isDebugEnabled()) {
+					LOG.debug(String.format("Column[%s] is to be excluded from audit: old value [%s], new
value[%s]", columnName, value, result));
+				}
+				result = null;
+			} else {
+				if (value.length() > limit) {
+					if (limit <= TruncationMarkerLength) {
+						// NOTE: If value is to be truncated to a size that is less than of equal to the Truncation
Marker then we won't put the marker in!!
+						result = value.substring(0, limit);
+					} else {
+						StringBuilder sb = new StringBuilder(value.substring(0, limit - TruncationMarkerLength));
+						sb.append(TruncationMarker);
+						result = sb.toString();
+					}
+					if (LOG.isDebugEnabled()) {
+						LOG.debug(String.format("Truncating value for column[%s] to [%d] characters: old value
[%s], new value[%s]", columnName, limit, value, result));
+					}
+				}
+			}
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug(String.format("<== getTrunctedValue(%s, %d, %s): %s", value, limit, columnName,
result));
+		}
+		return result;
+	}
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3b3b8d95/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
index 2c6a87f..2a07e94 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuditEventBase.java
@@ -21,6 +21,8 @@ package org.apache.ranger.audit.model;
 
 import java.util.Date;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.audit.dao.DaoManager;
 
 public abstract class AuditEventBase {
@@ -34,16 +36,4 @@ public abstract class AuditEventBase {
 	public abstract Date getEventTime ();
 	public abstract void setEventCount(long eventCount);
 	public abstract void setEventDurationMS(long eventDurationMS);
-	
-	protected String trim(String str, int len) {
-		String ret = str;
-		if (str != null) {
-			if (str.length() > len) {
-				ret = str.substring(0, len);
-			}
-		}
-		return ret;
-	}
-
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3b3b8d95/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
index d648de3..2a8d792 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/model/AuthzAuditEvent.java
@@ -190,8 +190,8 @@ public class AuthzAuditEvent extends AuditEventBase {
 	}
 
 	/**
-	 * @param timeStamp
-	 *            the timeStamp to set
+	 * @param eventTime
+	 *            the eventTime to set
 	 */
 	public void setEventTime(Date eventTime) {
 		this.eventTime = eventTime;
@@ -245,9 +245,7 @@ public class AuthzAuditEvent extends AuditEventBase {
 	/**
 	 * @return the action
 	 */
-	public String getAction() {
-		return trim(action, MAX_ACTION_FIELD_SIZE);
-	}
+	public String getAction() { return action; }
 
 	/**
 	 * @param action
@@ -380,9 +378,7 @@ public class AuthzAuditEvent extends AuditEventBase {
 	/**
 	 * @return the requestData
 	 */
-	public String getRequestData() {
-		return trim(requestData, MAX_REQUEST_DATA_FIELD_SIZE);
-	}
+	public String getRequestData() { return requestData; }
 
 	/**
 	 * @param requestData

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/3b3b8d95/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
index 98da1c2..f23f17d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/DbAuditProvider.java
@@ -32,6 +32,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.audit.dao.DaoManager;
 import org.apache.ranger.audit.destination.AuditDestination;
+import org.apache.ranger.audit.entity.AuthzAuditEventDbObj;
 import org.apache.ranger.audit.model.AuditEventBase;
 import org.apache.ranger.audit.model.AuthzAuditEvent;
 import org.apache.ranger.authorization.hadoop.utils.RangerCredentialProvider;
@@ -91,6 +92,9 @@ public class DbAuditProvider extends AuditDestination {
 		if(jdbcPassword != null && !jdbcPassword.isEmpty()) {
 			mDbProperties.put(AUDIT_JPA_JDBC_PASSWORD, jdbcPassword);
 		}
+
+		// initialize the database related classes
+		AuthzAuditEventDbObj.init(props);
 	}
 
 	@Override


Mime
View raw message