ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mad...@apache.org
Subject [3/9] incubator-ranger git commit: RANGER-1146: Policy engine optimization: dynamic reordering of policy evaluation order
Date Thu, 15 Sep 2016 19:52:07 GMT
RANGER-1146: Policy engine optimization: dynamic reordering of policy evaluation order

Signed-off-by: Madhan Neethiraj <madhan@apache.org>
(cherry picked from commit 3a36875e96aa2f689edd71ab77a8c74a6355493a)


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/efe49a88
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/efe49a88
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/efe49a88

Branch: refs/heads/ranger-0.6
Commit: efe49a88baa4ca13ba40299387afc11c769e5131
Parents: caba857
Author: Abhay Kulkarni <akulkarni@hortonworks.com>
Authored: Sat Aug 6 13:08:06 2016 -0700
Committer: Madhan Neethiraj <madhan@apache.org>
Committed: Thu Sep 15 12:19:12 2016 -0700

----------------------------------------------------------------------
 .../plugin/policyengine/RangerAccessResult.java |  26 +++
 .../plugin/policyengine/RangerPolicyEngine.java |   2 +
 .../policyengine/RangerPolicyEngineImpl.java    | 170 +++++++++++++++++--
 .../policyengine/RangerPolicyRepository.java    | 104 +++++++++---
 .../RangerAbstractPolicyEvaluator.java          |  36 +++-
 .../RangerDefaultPolicyEvaluator.java           |   7 +-
 .../RangerOptimizedPolicyEvaluator.java         |   5 -
 .../policyevaluator/RangerPolicyEvaluator.java  |   9 +
 .../ranger/plugin/service/RangerBasePlugin.java |  64 ++++++-
 .../ranger/plugin/util/PerfDataRecorder.java    |  41 ++---
 .../ranger/plugin/util/RangerPerfTracer.java    |   9 +
 .../plugin/policyengine/TestPolicyEngine.java   |   4 +
 .../ranger/policyengine/CommandLineParser.java  |  11 +-
 .../ranger/policyengine/PerfTestEngine.java     |  17 +-
 .../ranger/policyengine/PerfTestOptions.java    |   9 +-
 .../RangerPolicyenginePerfTester.java           |  32 ++--
 .../test/resources/testdata/test_modules.txt    |   3 +-
 ranger-tools/testdata/test_modules.txt          |   4 +-
 18 files changed, 465 insertions(+), 88 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
index 49d32b2..5543848 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerAccessResult.java
@@ -31,7 +31,9 @@ public class RangerAccessResult {
 	private boolean  isAllowed = false;
 	private boolean isAuditedDetermined = false;
 	private boolean  isAudited = false;
+	private long     auditPolicyId  = -1;
 	private long     policyId  = -1;
+	private long     evaluatedPoliciesCount = 0;
 	private String   reason    = null;
 
 	public RangerAccessResult(final String serviceName, final RangerServiceDef serviceDef, final RangerAccessRequest request) {
@@ -42,7 +44,9 @@ public class RangerAccessResult {
 		this.isAllowed   = false;
 		this.isAuditedDetermined = false;
 		this.isAudited   = false;
+		this.auditPolicyId = -1;
 		this.policyId    = -1;
+		this.evaluatedPoliciesCount = 0;
 		this.reason      = null;
 	}
 
@@ -50,12 +54,15 @@ public class RangerAccessResult {
 		this.isAccessDetermined = other.getIsAccessDetermined();
 		this.isAllowed   = other.getIsAllowed();
 		this.policyId    = other.getPolicyId();
+		this.evaluatedPoliciesCount = other.evaluatedPoliciesCount;
 		this.reason      = other.getReason();
 	}
 
 	public void setAuditResultFrom(final RangerAccessResult other) {
 		this.isAuditedDetermined = other.getIsAuditedDetermined();
 		this.isAudited = other.getIsAudited();
+		this.auditPolicyId = other.getAuditPolicyId();
+		this.evaluatedPoliciesCount += other.getEvaluatedPoliciesCount();
 	}
 
 	/**
@@ -144,12 +151,29 @@ public class RangerAccessResult {
 	}
 
 	/**
+	 * @return the auditPolicyId
+	 */
+	public long getAuditPolicyId() {
+		return auditPolicyId;
+	}
+
+	public long getEvaluatedPoliciesCount() { return this.evaluatedPoliciesCount; }
+	/**
 	 * @param policyId the policyId to set
 	 */
 	public void setPolicyId(long policyId) {
 		this.policyId = policyId;
 	}
 
+	/**
+	 * @param policyId the auditPolicyId to set
+	 */
+	public void setAuditPolicyId(long policyId) {
+		this.auditPolicyId = policyId;
+	}
+
+	public void incrementEvaluatedPoliciesCount() { this.evaluatedPoliciesCount++; }
+
 	public int getServiceType() {
 		int ret = -1;
 
@@ -177,6 +201,8 @@ public class RangerAccessResult {
         sb.append("isAuditedDetermined={").append(isAuditedDetermined).append("} ");
         sb.append("isAudited={").append(isAudited).append("} ");
 		sb.append("policyId={").append(policyId).append("} ");
+		sb.append("auditPolicyId={").append(auditPolicyId).append("} ");
+		sb.append("evaluatedPoliciesCount={").append(evaluatedPoliciesCount).append("} ");
 		sb.append("reason={").append(reason).append("} ");
 
 		sb.append("}");

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
index 3e69d03..a5e92da 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngine.java
@@ -69,6 +69,8 @@ public interface RangerPolicyEngine {
 
 	RangerResourceAccessInfo getResourceAccessInfo(RangerAccessRequest request);
 
+	void reorderPolicyEvaluators();
+
 	boolean preCleanup();
 
 	void setUseForwardedIPAddress(boolean useForwardedIPAddress);

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
index 8d3deb6..661d54f 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyEngineImpl.java
@@ -20,6 +20,7 @@
 package org.apache.ranger.plugin.policyengine;
 
 import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -34,7 +35,13 @@ import org.apache.ranger.plugin.util.RangerAccessRequestUtil;
 import org.apache.ranger.plugin.util.RangerPerfTracer;
 import org.apache.ranger.plugin.util.ServicePolicies;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 	private static final Log LOG = LogFactory.getLog(RangerPolicyEngineImpl.class);
@@ -43,6 +50,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 	private static final Log PERF_POLICYENGINE_REQUEST_LOG = RangerPerfTracer.getPerfLogger("policyengine.request");
 	private static final Log PERF_POLICYENGINE_AUDIT_LOG = RangerPerfTracer.getPerfLogger("policyengine.audit");
 	private static final Log PERF_CONTEXTENRICHER_REQUEST_LOG = RangerPerfTracer.getPerfLogger("contextenricher.request");
+	private static final Log PERF_POLICYENGINE_REBALANCE_LOG = RangerPerfTracer.getPerfLogger("policyengine.rebalance");
+	private static final Log PERF_POLICYENGINE_USAGE_LOG = RangerPerfTracer.getPerfLogger("policyengine.usage");
 
 	private static final int MAX_POLICIES_FOR_CACHE_TYPE_EVALUATOR = 100;
 
@@ -51,6 +60,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 	
 	private List<RangerContextEnricher> allContextEnrichers;
 
+	private final Map<Long, RangerPolicyEvaluator> policyEvaluatorsMap;
+
 	private boolean  useForwardedIPAddress = false;
 	private String[] trustedProxyAddresses = null;
 
@@ -132,6 +143,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 
 		this.allContextEnrichers = tmpList;
 
+		policyEvaluatorsMap = createPolicyEvaluatorsMap();
+
 		RangerPerfTracer.log(perf);
 
 		if (PERF_POLICYENGINE_INIT_LOG.isDebugEnabled()) {
@@ -160,6 +173,15 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 		return policyRepository.getPolicyVersion();
 	}
 
+	public RangerPolicyEvaluator getPolicyEvaluator(Long id) {
+		return policyEvaluatorsMap.get(id);
+	}
+
+	public RangerPolicy getPolicy(Long id) {
+		RangerPolicyEvaluator evaluator = getPolicyEvaluator(id);
+		return evaluator != null ? evaluator.getPolicy() : null;
+	}
+
 	@Override
 	public RangerAccessResult createAccessResult(RangerAccessRequest request) {
 		return new RangerAccessResult(this.getServiceName(), policyRepository.getServiceDef(), request);
@@ -231,6 +253,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 
 		RangerAccessResult ret = isAccessAllowedNoAudit(request);
 
+		updatePolicyUsageCounts(request, ret);
+
 		if (resultProcessor != null) {
 
 			RangerPerfTracer perfAuditTracer = null;
@@ -309,6 +333,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 			ret.setIsAudited(false);
 		}
 
+		updatePolicyUsageCounts(request, ret);
+
 		if (resultProcessor != null) {
 			resultProcessor.processResult(ret);
 		}
@@ -348,6 +374,8 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 			ret.setIsAudited(false);
 		}
 
+		updatePolicyUsageCounts(request, ret);
+
 		if (resultProcessor != null) {
 			resultProcessor.processResult(ret);
 		}
@@ -570,6 +598,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 
 				List<RangerPolicyEvaluator> evaluators = policyRepository.getPolicyEvaluators();
 				for (RangerPolicyEvaluator evaluator : evaluators) {
+					ret.incrementEvaluatedPoliciesCount();
 					evaluator.evaluate(request, ret);
 
 					if(ret.getIsAllowed() && !evaluator.hasDeny()) { // all policies having deny have been evaluated
@@ -626,6 +655,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 					tagEvalResult.setAuditResultFrom(result);
 
 					for (RangerPolicyEvaluator evaluator : evaluators) {
+						tagEvalResult.incrementEvaluatedPoliciesCount();
 						if(! evaluator.isMatch(tagEvalRequest.getResource())) 
 							continue;
 
@@ -650,6 +680,7 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 
 					if (tagEvalResult.getIsAudited()) {
 						result.setIsAudited(true);
+						result.setAuditPolicyId(tagEvalResult.getAuditPolicyId());
 					}
 
 					if(!result.getIsAccessDetermined() && tagEvalResult.getIsAccessDetermined()) {
@@ -676,25 +707,40 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 		}
 	}
 
-	private void setResourceServiceDef(RangerAccessRequest request) {
-		RangerAccessResource resource = request.getResource();
+	@Override
+	public void reorderPolicyEvaluators() {
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("==> reorderPolicyEvaluators()");
+		}
+		RangerPerfTracer perf = null;
 
-		if (resource.getServiceDef() == null) {
-			if (resource instanceof RangerMutableResource) {
-				RangerMutableResource mutable = (RangerMutableResource) resource;
-				mutable.setServiceDef(getServiceDef());
-			} else {
-				LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerTagResourceMap.");
+		if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_REBALANCE_LOG)) {
+			perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_REBALANCE_LOG, "RangerPolicyEngine.reorderPolicyEvaluators()");
+		}
+		if (MapUtils.isNotEmpty(policyEvaluatorsMap)) {
+			for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) {
+				entry.getValue().setUsageCountImmutable();
 			}
 		}
-	}
 
-	private boolean hasTagPolicies() {
-		return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies());
-	}
+		if (tagPolicyRepository != null) {
+			tagPolicyRepository.reorderPolicyEvaluators();
+		}
+		if (policyRepository != null) {
+			policyRepository.reorderPolicyEvaluators();
+		}
 
-	private boolean hasResourcePolicies() {
-		return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies());
+		if (MapUtils.isNotEmpty(policyEvaluatorsMap)) {
+			for (Map.Entry<Long, RangerPolicyEvaluator> entry : policyEvaluatorsMap.entrySet()) {
+				entry.getValue().resetUsageCount();
+			}
+		}
+
+		RangerPerfTracer.log(perf);
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug("<== reorderPolicyEvaluators()");
+		}
 	}
 
 	@Override
@@ -792,6 +838,100 @@ public class RangerPolicyEngineImpl implements RangerPolicyEngine {
 
 		return sb;
 	}
+
+
+	private void setResourceServiceDef(RangerAccessRequest request) {
+		RangerAccessResource resource = request.getResource();
+
+		if (resource.getServiceDef() == null) {
+			if (resource instanceof RangerMutableResource) {
+				RangerMutableResource mutable = (RangerMutableResource) resource;
+				mutable.setServiceDef(getServiceDef());
+			} else {
+				LOG.debug("RangerPolicyEngineImpl.setResourceServiceDef(): Cannot set ServiceDef in RangerTagResourceMap.");
+			}
+		}
+	}
+
+	private boolean hasTagPolicies() {
+		return tagPolicyRepository != null && CollectionUtils.isNotEmpty(tagPolicyRepository.getPolicies());
+	}
+
+	private boolean hasResourcePolicies() {
+		return policyRepository != null && CollectionUtils.isNotEmpty(policyRepository.getPolicies());
+	}
+
+	private Map<Long, RangerPolicyEvaluator> createPolicyEvaluatorsMap() {
+		Map<Long, RangerPolicyEvaluator> tmpPolicyEvaluatorMap = new HashMap<Long, RangerPolicyEvaluator>();
+
+		if (tagPolicyRepository != null) {
+			for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getPolicyEvaluators()) {
+				tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+			}
+			for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getDataMaskPolicyEvaluators()) {
+				tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+			}
+			for (RangerPolicyEvaluator evaluator : tagPolicyRepository.getRowFilterPolicyEvaluators()) {
+				tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+			}
+		}
+		for (RangerPolicyEvaluator evaluator : policyRepository.getPolicyEvaluators()) {
+			tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+		}
+		for (RangerPolicyEvaluator evaluator : policyRepository.getDataMaskPolicyEvaluators()) {
+			tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+		}
+		for (RangerPolicyEvaluator evaluator : policyRepository.getRowFilterPolicyEvaluators()) {
+			tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator);
+		}
+
+		return  Collections.unmodifiableMap(tmpPolicyEvaluatorMap);
+	}
+
+	private void updatePolicyUsageCounts(RangerAccessRequest accessRequest, RangerAccessResult accessResult) {
+
+		boolean auditCountUpdated = false;
+
+		if (accessResult.getIsAccessDetermined()) {
+			RangerPolicyEvaluator accessPolicy = getPolicyEvaluator(accessResult.getPolicyId());
+
+			if (accessPolicy != null) {
+
+				if (accessPolicy.getPolicy().getIsAuditEnabled()) {
+					updateUsageCount(accessPolicy, 2);
+					accessResult.setAuditPolicyId(accessResult.getPolicyId());
+
+					auditCountUpdated = true;
+				} else {
+					updateUsageCount(accessPolicy, 1);
+				}
+
+			}
+		}
+
+		if (!auditCountUpdated && accessResult.getIsAuditedDetermined()) {
+			long auditPolicyId = accessResult.getAuditPolicyId();
+			RangerPolicyEvaluator auditPolicy = auditPolicyId == -1 ? null : getPolicyEvaluator(auditPolicyId);
+
+			updateUsageCount(auditPolicy, 1);
+		}
+
+		if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICYENGINE_USAGE_LOG)) {
+			RangerAccessRequestImpl rangerAccessRequest = (RangerAccessRequestImpl) accessRequest;
+			RangerPerfTracer perf = RangerPerfTracer.getPerfTracer(PERF_POLICYENGINE_USAGE_LOG,
+					"RangerPolicyEngine.usage(accessingUser=" + rangerAccessRequest.getUser()
+							+ ",accessedResource=" + rangerAccessRequest.getResource().getAsString()
+							+ ",accessType=" + rangerAccessRequest.getAccessType()
+							+ ",evaluatedPoliciesCount=" + accessResult.getEvaluatedPoliciesCount() + ")");
+			RangerPerfTracer.logAlways(perf);
+		}
+	}
+
+	private void updateUsageCount(RangerPolicyEvaluator evaluator, int number) {
+		if (evaluator != null) {
+			evaluator.incrementUsageCount(number);
+		}
+	}
 }
 class RangerTagResource extends RangerAccessResourceImpl {
 	private static final String KEY_TAG = "tag";

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
index 01a547c..0012329 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyengine/RangerPolicyRepository.java
@@ -42,7 +42,7 @@ import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
-public class RangerPolicyRepository {
+class RangerPolicyRepository {
     private static final Log LOG = LogFactory.getLog(RangerPolicyRepository.class);
 
     private static final Log PERF_CONTEXTENRICHER_INIT_LOG = RangerPerfTracer.getPerfLogger("contextenricher.init");
@@ -51,6 +51,25 @@ public class RangerPolicyRepository {
         AUDIT_ALL, AUDIT_NONE, AUDIT_DEFAULT
     }
 
+    class AuditInfo {
+        final boolean isAudited;
+        final long    auditPolicyId;
+
+        AuditInfo() {
+            this(false, -1);
+        }
+        AuditInfo(boolean isAudited, long auditPolicyId) {
+            this.isAudited = isAudited;
+            this.auditPolicyId = auditPolicyId;
+        }
+        long getAuditPolicyId() {
+            return this.auditPolicyId;
+        }
+        boolean getIsAudited() {
+            return isAudited;
+        }
+    }
+
     private final String                      serviceName;
     private final String                      appId;
     private final RangerServiceDef            serviceDef;
@@ -61,7 +80,7 @@ public class RangerPolicyRepository {
     private List<RangerPolicyEvaluator>       dataMaskPolicyEvaluators;
     private List<RangerPolicyEvaluator>       rowFilterPolicyEvaluators;
     private final AuditModeEnum               auditModeEnum;
-    private final Map<String, Boolean>        accessAuditCache;
+    private final Map<String, AuditInfo>      accessAuditCache;
 
     private final String                      componentServiceName;
     private final RangerServiceDef            componentServiceDef;
@@ -98,7 +117,7 @@ public class RangerPolicyRepository {
                 final int RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE = 64 * 1024;
 
                 int auditResultCacheSize = RangerConfiguration.getInstance().getInt(propertyName, RANGER_POLICYENGINE_AUDITRESULT_CACHE_SIZE);
-                accessAuditCache = Collections.synchronizedMap(new CacheMap<String, Boolean>(auditResultCacheSize));
+                accessAuditCache = Collections.synchronizedMap(new CacheMap<String, AuditInfo>(auditResultCacheSize));
             } else {
                 accessAuditCache = null;
             }
@@ -166,15 +185,15 @@ public class RangerPolicyRepository {
 
     public List<RangerContextEnricher> getContextEnrichers() { return contextEnrichers; }
 
-    public List<RangerPolicyEvaluator> getPolicyEvaluators() {
+    List<RangerPolicyEvaluator> getPolicyEvaluators() {
         return policyEvaluators;
     }
 
-    public List<RangerPolicyEvaluator> getDataMaskPolicyEvaluators() {
+    List<RangerPolicyEvaluator> getDataMaskPolicyEvaluators() {
         return dataMaskPolicyEvaluators;
     }
 
-    public List<RangerPolicyEvaluator> getRowFilterPolicyEvaluators() {
+    List<RangerPolicyEvaluator> getRowFilterPolicyEvaluators() {
         return rowFilterPolicyEvaluators;
     }
 
@@ -323,10 +342,8 @@ public class RangerPolicyRepository {
         return policyItems;
     }
 
-    public static boolean isDelegateAdminPolicy(RangerPolicy policy) {
-        boolean ret = false;
-
-        ret =      hasDelegateAdminItems(policy.getPolicyItems())
+    private static boolean isDelegateAdminPolicy(RangerPolicy policy) {
+        boolean ret =      hasDelegateAdminItems(policy.getPolicyItems())
                 || hasDelegateAdminItems(policy.getDenyPolicyItems())
                 || hasDelegateAdminItems(policy.getAllowExceptions())
                 || hasDelegateAdminItems(policy.getDenyExceptions());
@@ -510,42 +527,49 @@ public class RangerPolicyRepository {
             LOG.debug("==> RangerPolicyRepository.setAuditEnabledFromCache()");
         }
 
-        Boolean value = null;
+        final boolean auditResult;
+        final boolean foundInCache;
 
         switch (auditModeEnum) {
             case AUDIT_ALL:
-                value = Boolean.TRUE;
+                auditResult = true;
+                foundInCache = true;
                 break;
             case AUDIT_NONE:
-                value = Boolean.FALSE;
+                auditResult = false;
+                foundInCache = true;
                 break;
             default:
-                if (accessAuditCache != null) {
-                    value = accessAuditCache.get(request.getResource().getAsString());
+                AuditInfo auditInfo = accessAuditCache != null ? accessAuditCache.get(request.getResource().getAsString()) : null;
+                if (auditInfo != null) {
+                    auditResult = auditInfo.getIsAudited();
+                    result.setAuditPolicyId(auditInfo.getAuditPolicyId());
+                    foundInCache = true;
+                } else {
+                    auditResult = false;
+                    foundInCache = false;
                 }
                 break;
         }
 
-        if ((value != null)) {
-            result.setIsAudited(value);
+        if (foundInCache) {
+            result.setIsAudited(auditResult);
         }
 
         if (LOG.isDebugEnabled()) {
             LOG.debug("<== RangerPolicyRepository.setAuditEnabledFromCache()");
         }
 
-        return value != null;
+        return foundInCache;
     }
 
-     void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult ret) {
+     void storeAuditEnabledInCache(RangerAccessRequest request, RangerAccessResult result) {
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> RangerPolicyRepository.storeAuditEnabledInCache()");
         }
 
-        if (accessAuditCache != null && ret.getIsAuditedDetermined()) {
-            String strResource = request.getResource().getAsString();
-            Boolean value = ret.getIsAudited() ? Boolean.TRUE : Boolean.FALSE;
-            accessAuditCache.put(strResource, value);
+        if (accessAuditCache != null && result.getIsAuditedDetermined()) {
+            accessAuditCache.put(request.getResource().getAsString(), new AuditInfo(result.getIsAudited(), result.getAuditPolicyId()));
         }
 
         if (LOG.isDebugEnabled()) {
@@ -553,11 +577,7 @@ public class RangerPolicyRepository {
         }
     }
 
-    /**
-     * Remove nulls from policy resource values
-     * @param policy
-     */
-    boolean scrubPolicy(RangerPolicy policy) {
+    private boolean scrubPolicy(RangerPolicy policy) {
         if (LOG.isDebugEnabled()) {
             LOG.debug("==> RangerPolicyRepository.scrubPolicy(" + policy + ")");
         }
@@ -584,6 +604,34 @@ public class RangerPolicyRepository {
         return altered;
     }
 
+    void reorderPolicyEvaluators() {
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("==> reorderPolicyEvaluators()");
+        }
+
+        this.policyEvaluators = getReorderedPolicyEvaluators(this.policyEvaluators);
+        this.dataMaskPolicyEvaluators = getReorderedPolicyEvaluators(this.dataMaskPolicyEvaluators);
+        this.rowFilterPolicyEvaluators = getReorderedPolicyEvaluators(this.rowFilterPolicyEvaluators);
+
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("<== reorderPolicyEvaluators()");
+        }
+    }
+
+    private List<RangerPolicyEvaluator> getReorderedPolicyEvaluators(List<RangerPolicyEvaluator> evaluators) {
+        List<RangerPolicyEvaluator> ret = evaluators;
+
+        if (CollectionUtils.isNotEmpty(evaluators)) {
+
+            ret = new ArrayList<RangerPolicyEvaluator>(evaluators);
+            Collections.sort(ret);
+
+            ret = Collections.unmodifiableList(ret);
+        }
+
+        return ret;
+    }
+
     @Override
     public String toString( ) {
         StringBuilder sb = new StringBuilder();

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
index adc7d8c..8f45661 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerAbstractPolicyEvaluator.java
@@ -35,6 +35,8 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
 	private RangerPolicy     policy     = null;
 	private RangerServiceDef serviceDef = null;
 	private int              evalOrder  = 0;
+	protected long           usageCount = 0;
+	protected boolean        usageCountMutable = true;
 
 
 	@Override
@@ -74,10 +76,14 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
 	}
 
 	@Override
+	public long getUsageCount() {
+		return usageCount;
+	}
+
+	@Override
 	public int getEvalOrder() {
 		return evalOrder;
 	}
-
 	@Override
 	public boolean isAuditEnabled() {
 		return policy != null && policy.getIsAuditEnabled();
@@ -89,7 +95,17 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
 		LOG.debug("==> RangerAbstractPolicyEvaluator.compareTo()");
 		}
 
-		int result = Integer.compare(this.getEvalOrder(), other.getEvalOrder());
+		int result;
+		if (hasDeny() && !other.hasDeny()) {
+			result = -1;
+		} else if (!hasDeny() && other.hasDeny()) {
+			result = 1;
+		} else {
+			result = Long.compare(other.getUsageCount(), this.usageCount);
+			if (result == 0) {
+				result = Integer.compare(this.evalOrder, other.getEvalOrder());
+			}
+		}
 
 		if(LOG.isDebugEnabled()) {
 			LOG.debug("<== RangerAbstractPolicyEvaluator.compareTo(), result:" + result);
@@ -103,6 +119,22 @@ public abstract class RangerAbstractPolicyEvaluator implements RangerPolicyEvalu
 	}
 
 	@Override
+	public void incrementUsageCount(int number) {
+		if (usageCountMutable) usageCount += number;
+	}
+
+	@Override
+	public void setUsageCountImmutable() {
+		this.usageCountMutable = false;
+	}
+
+	@Override
+	public void resetUsageCount() {
+		this.usageCount = 0;
+		this.usageCountMutable = true;
+	}
+
+	@Override
 	public String toString( ) {
 		StringBuilder sb = new StringBuilder();
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
index 368a695..f954ccf 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerDefaultPolicyEvaluator.java
@@ -177,6 +177,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
                     // We are done for determining if audit is needed for this policy
                     if (isAuditEnabled()) {
                         result.setIsAudited(true);
+                        result.setAuditPolicyId(getPolicy().getId());
                     }
                 }
             }
@@ -249,6 +250,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
 				if (isResourceMatch || isResourceHeadMatch) {
 					if (isAuditEnabled()) {
 						result.setIsAudited(true);
+						result.setAuditPolicyId(getPolicy().getId());
 					}
 				}
 			}
@@ -270,6 +272,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
 					evaluatePolicyItems(request, result);
 				}
 			}
+
 		}
 
 		RangerPerfTracer.log(perf);
@@ -314,6 +317,7 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
 				if (isResourceMatch || isResourceHeadMatch) {
 					if (isAuditEnabled()) {
 						result.setIsAudited(true);
+						result.setAuditPolicyId(getPolicy().getId());
 					}
 				}
 			}
@@ -990,5 +994,6 @@ public class RangerDefaultPolicyEvaluator extends RangerAbstractPolicyEvaluator
         }
 
         return ret;
-	}
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java
index 2e777ae..00b24d1 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerOptimizedPolicyEvaluator.java
@@ -47,7 +47,6 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator
     private static final String RANGER_POLICY_EVAL_MATCH_ONE_CHARACTER_STRING = "?";
 
     private static final int RANGER_POLICY_EVAL_SCORE_DEFAULT                         = 10000;
-    private static final int RANGER_POLICY_EVAL_SCORE_DISCOUNT_POLICY_HAS_DENY        =  4000;
 
     private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_RESOURCE          = 100;
     private static final int RANGER_POLICY_EVAL_SCORE_MAX_DISCOUNT_USERSGROUPS       =  25;
@@ -202,10 +201,6 @@ public class RangerOptimizedPolicyEvaluator extends RangerDefaultPolicyEvaluator
             evalOrder -= customConditionsDiscount;
         }
 
-        if (hasDeny()) {
-            evalOrder -= RANGER_POLICY_EVAL_SCORE_DISCOUNT_POLICY_HAS_DENY;
-        }
-
         if(LOG.isDebugEnabled()) {
             LOG.debug("<== RangerOptimizedPolicyEvaluator.computeEvalOrder(), policyName:" + policy.getName() + ", priority:" + evalOrder);
         }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
index be97830..a7dc833 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/policyevaluator/RangerPolicyEvaluator.java
@@ -52,6 +52,14 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator>
 
 	int getEvalOrder();
 
+	long getUsageCount();
+
+	void incrementUsageCount(int number);
+
+	void setUsageCountImmutable();
+
+	void resetUsageCount();
+
 	int getCustomConditionsCount();
 
 	boolean isAuditEnabled();
@@ -73,4 +81,5 @@ public interface RangerPolicyEvaluator extends Comparable<RangerPolicyEvaluator>
 	boolean isAccessAllowed(Map<String, RangerPolicyResource> resources, String user, Set<String> userGroups, String accessType);
 
 	void getResourceAccessInfo(RangerAccessRequest request, RangerResourceAccessInfo result);
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
index a000125..760c4a3 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/service/RangerBasePlugin.java
@@ -22,6 +22,8 @@ package org.apache.ranger.plugin.service;
 import java.util.Collection;
 import java.util.Hashtable;
 import java.util.Map;
+import java.util.Timer;
+import java.util.TimerTask;
 
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
@@ -30,7 +32,17 @@ import org.apache.ranger.admin.client.RangerAdminClient;
 import org.apache.ranger.admin.client.RangerAdminRESTClient;
 import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
 import org.apache.ranger.plugin.model.RangerServiceDef;
-import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequest;
+import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl;
+import org.apache.ranger.plugin.policyengine.RangerAccessResult;
+import org.apache.ranger.plugin.policyengine.RangerAccessResultProcessor;
+import org.apache.ranger.plugin.policyengine.RangerDataMaskResult;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngine;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngineImpl;
+import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions;
+import org.apache.ranger.plugin.policyengine.RangerResourceAccessInfo;
+import org.apache.ranger.plugin.policyengine.RangerRowFilterResult;
 import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
 import org.apache.ranger.plugin.util.GrantRevokeRequest;
 import org.apache.ranger.plugin.util.PolicyRefresher;
@@ -51,6 +63,7 @@ public class RangerBasePlugin {
 	private RangerAccessResultProcessor resultProcessor = null;
 	private boolean                   useForwardedIPAddress = false;
 	private String[]                  trustedProxyAddresses = null;
+	private Timer                     policyEngineRefreshTimer;
 
 	Map<String, LogHistory> logHistoryList = new Hashtable<String, RangerBasePlugin.LogHistory>();
 	int logInterval = 30000; // 30 seconds
@@ -126,6 +139,32 @@ public class RangerBasePlugin {
 		refresher = new PolicyRefresher(this, serviceType, appId, serviceName, admin, pollingIntervalMs, cacheDir);
 		refresher.setDaemon(true);
 		refresher.startRefresher();
+
+		long policyReorderIntervalMs = RangerConfiguration.getInstance().getLong(propertyPrefix + ".policy.policyReorderInterval", 60 * 1000);
+		if (policyReorderIntervalMs >= 0 && policyReorderIntervalMs < 15 * 1000) {
+			policyReorderIntervalMs = 15 * 1000;
+		}
+
+		if (LOG.isDebugEnabled()) {
+			LOG.debug(propertyPrefix + ".policy.policyReorderInterval:" + policyReorderIntervalMs);
+		}
+
+		if (policyReorderIntervalMs > 0) {
+			policyEngineRefreshTimer = new Timer("PolicyEngineRefreshTimer", true);
+			try {
+				policyEngineRefreshTimer.schedule(new PolicyEngineRefresher(this), policyReorderIntervalMs, policyReorderIntervalMs);
+				if (LOG.isDebugEnabled()) {
+					LOG.debug("Scheduled PolicyEngineRefresher to reorder policies nbased on number of evaluations in and every " + policyReorderIntervalMs + " milliseconds");
+				}
+			} catch (IllegalStateException exception) {
+				LOG.error("Error scheduling policyEngineRefresher:", exception);
+				LOG.error("*** PolicyEngine will NOT be reorderd based on number of evaluations every " + policyReorderIntervalMs + " milliseconds ***");
+				policyEngineRefreshTimer = null;
+			}
+		} else {
+			LOG.info("Policies will NOT be reordered based on number of evaluations because "
+					+ propertyPrefix + ".policy.policyReorderInterval is set to a negative number[" + policyReorderIntervalMs +"]");
+		}
 	}
 
 	public void setPolicies(ServicePolicies policies) {
@@ -154,17 +193,25 @@ public class RangerBasePlugin {
 
 		RangerPolicyEngine policyEngine = this.policyEngine;
 
+		Timer policyEngineRefreshTimer = this.policyEngineRefreshTimer;
+
 		this.serviceName  = null;
 		this.policyEngine = null;
 		this.refresher    = null;
+		this.policyEngineRefreshTimer = null;
 
 		if (refresher != null) {
 			refresher.stopRefresher();
 		}
 
+		if (policyEngineRefreshTimer != null) {
+			policyEngineRefreshTimer.cancel();
+		}
+
 		if (policyEngine != null) {
 			policyEngine.cleanup();
 		}
+
 	}
 
 	public void setResultProcessor(RangerAccessResultProcessor resultProcessor) {
@@ -400,4 +447,19 @@ public class RangerBasePlugin {
 		int counter=0;
 	}
 
+	private class PolicyEngineRefresher extends TimerTask {
+		private final RangerBasePlugin plugin;
+
+		PolicyEngineRefresher(RangerBasePlugin plugin) {
+			this.plugin = plugin;
+		}
+
+		@Override
+		public void run() {
+			RangerPolicyEngine policyEngine = plugin.policyEngine;
+			if (policyEngine != null) {
+				policyEngine.reorderPolicyEvaluators();
+			}
+		}
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
index ddc42c0..3fe96ac 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PerfDataRecorder.java
@@ -36,7 +36,6 @@ public class PerfDataRecorder {
 
 	private static volatile PerfDataRecorder instance = null;
 	private Map<String, PerfStatistic> perfStatistics = new HashMap<String, PerfStatistic>();
-	private boolean initPerfStatisticsOnce = true;
 
 	public static void initialize(List<String> names) {
 		if (instance == null) {
@@ -57,6 +56,13 @@ public class PerfDataRecorder {
 			instance.dumpStatistics();
 		}
 	}
+
+	public static void clearStatistics() {
+		if (instance != null) {
+			instance.clear();
+		}
+	}
+
 	public static void recordStatistic(String tag, long elapsedTime) {
 		if (instance != null) {
 			instance.record(tag, elapsedTime);
@@ -73,19 +79,16 @@ public class PerfDataRecorder {
 			PerfStatistic perfStatistic = perfStatistics.get(tag);
 
 			long averageTimeSpent = 0L;
-			long minTimeSpent = 0L;
-			long maxTimeSpent = 0L;
+
 			if (perfStatistic.numberOfInvocations.get() != 0L) {
 				averageTimeSpent = perfStatistic.millisecondsSpent.get()/perfStatistic.numberOfInvocations.get();
-				minTimeSpent = perfStatistic.minTimeSpent.get();
-				maxTimeSpent = perfStatistic.maxTimeSpent.get();
 			}
 
 			String logMsg = "[" + tag + "]" +
-                             " execCount:" + perfStatistic.numberOfInvocations +
-                             ", totalTimeTaken:" + perfStatistic.millisecondsSpent +
-                             ", maxTimeTaken:" + maxTimeSpent +
-                             ", minTimeTaken:" + minTimeSpent +
+                             " execCount:" + perfStatistic.numberOfInvocations.get() +
+                             ", totalTimeTaken:" + perfStatistic.millisecondsSpent.get() +
+                             ", maxTimeTaken:" + perfStatistic.maxTimeSpent.get() +
+                             ", minTimeTaken:" + perfStatistic.minTimeSpent.get() +
                              ", avgTimeTaken:" + averageTimeSpent;
 
 			LOG.info(logMsg);
@@ -93,23 +96,25 @@ public class PerfDataRecorder {
 		}
 	}
 
+	private void clear() {
+		perfStatistics.clear();
+	}
+
 	private void record(String tag, long elapsedTime) {
 		PerfStatistic perfStatistic = perfStatistics.get(tag);
 
-		if (perfStatistic == null  && !initPerfStatisticsOnce) {
+		if (perfStatistic == null) {
 			synchronized (PerfDataRecorder.class) {
 				perfStatistic = perfStatistics.get(tag);
-				if (perfStatistic == null) {
+
+				if(perfStatistic == null) {
 					perfStatistic = new PerfStatistic();
 					perfStatistics.put(tag, perfStatistic);
 				}
 			}
 		}
 
-		if (perfStatistic != null) {
-			perfStatistic.addPerfDataItem(elapsedTime);
-		}
-
+		perfStatistic.addPerfDataItem(elapsedTime);
 	}
 
 	private PerfDataRecorder(List<String> names) {
@@ -118,8 +123,6 @@ public class PerfDataRecorder {
 				// Create structure
 				perfStatistics.put(name, new PerfStatistic());
 			}
-		} else {
-			initPerfStatisticsOnce = false;
 		}
 	}
 
@@ -134,12 +137,12 @@ public class PerfDataRecorder {
 			millisecondsSpent.getAndAdd(timeTaken);
 
 			long min = minTimeSpent.get();
-			if (timeTaken < min) {
+			if(timeTaken < min) {
 				minTimeSpent.compareAndSet(min, timeTaken);
 			}
 
 			long max = maxTimeSpent.get();
-			if (timeTaken > max) {
+			if(timeTaken > max) {
 				maxTimeSpent.compareAndSet(max, timeTaken);
 			}
 		}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
index e130cc7..4b17110 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerPerfTracer.java
@@ -71,6 +71,11 @@ public class RangerPerfTracer {
 		}
 	}
 
+	public static void logAlways(RangerPerfTracer tracer) {
+		if(tracer != null) {
+			tracer.logAlways();
+		}
+	}
 	public RangerPerfTracer(Log logger, String tag, String data) {
 		this.logger = logger;
 		this.tag    = tag;
@@ -96,4 +101,8 @@ public class RangerPerfTracer {
 			logger.debug("[PERF] " + tag + data + ": " + elapsedTime);
 		}
 	}
+	public void logAlways() {
+		long elapsedTime = getElapsedTime();
+		logger.debug("[PERF] " + tag + data + ": " + elapsedTime);
+	}
 }

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
----------------------------------------------------------------------
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
index 1d03ef8..ab5626e 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/policyengine/TestPolicyEngine.java
@@ -300,11 +300,15 @@ public class TestPolicyEngine {
 		policyEngine = new RangerPolicyEngineImpl(testName, servicePolicies, policyEngineOptions);
 		policyEngine.setUseForwardedIPAddress(useForwardedIPAddress);
 		policyEngine.setTrustedProxyAddresses(trustedProxyAddresses);
+		long requestCount = 0L;
 
 		RangerAccessRequest request = null;
 
 		for(TestData test : testCase.tests) {
 			request = test.request;
+			if ((requestCount++ % 10) == 1) {
+				policyEngine.reorderPolicyEvaluators();
+			}
 			if (request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_TAGS) ||
 					request.getContext().containsKey(RangerAccessRequestUtil.KEY_CONTEXT_REQUESTED_RESOURCES)) {
 				// Create a new AccessRequest

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
index ab7d532..d6c028f 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/CommandLineParser.java
@@ -52,6 +52,8 @@ public class CommandLineParser
     private int concurrentClientCount = 1;
     private int iterationsCount = 1;
 
+    private boolean isDynamicReorderingEnabled = false;
+
     private Options options = new Options();
 
     CommandLineParser() {}
@@ -60,7 +62,7 @@ public class CommandLineParser
         PerfTestOptions ret = null;
         if (parseArguments(args) && validateInputFiles()) {
             // Instantiate a data-object and return
-            ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount);
+            ret = new PerfTestOptions(servicePoliciesFileURL, requestFileURLs, statCollectionFileURL, concurrentClientCount, iterationsCount, isDynamicReorderingEnabled);
         } else {
             showUsage();
         }
@@ -75,6 +77,7 @@ public class CommandLineParser
             -r request-file-name-list
             -n number-of-iterations
             -p modules-to-collect-stats
+            -o
 
             If the concurrent-client-count is more than the number of files in the request-file-name-list,
             then reuse the request-file-names in a round-robin way
@@ -94,6 +97,7 @@ public class CommandLineParser
         options.addOption("p", "statistics", true, "Modules for stat collection File Name");
         options.addOption("c", "clients", true, "Number of concurrent clients");
         options.addOption("n", "cycles", true, "Number of iterations");
+        options.addOption("o", "optimize", false, "Enable usage-based policy reordering");
 
         org.apache.commons.cli.CommandLineParser commandLineParser = new DefaultParser();
 
@@ -120,9 +124,14 @@ public class CommandLineParser
             if (iterationsOptionValue != null) {
                 iterationsCount = Integer.parseInt(iterationsOptionValue);
             }
+            if (commandLine.hasOption("o")) {
+                isDynamicReorderingEnabled = true;
+            }
+
             if (LOG.isDebugEnabled()) {
                 LOG.debug("servicePoliciesFileName=" + servicePoliciesFileName + ", requestFileName=" + Arrays.toString(requestFileNames));
                 LOG.debug("concurrentClientCount=" + concurrentClientCount + ", iterationsCount=" + iterationsCount);
+                LOG.debug("isDynamicReorderingEnabled=" + isDynamicReorderingEnabled);
             }
 
             ret = true;

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
index 38c6e2f..cf83260 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestEngine.java
@@ -24,6 +24,7 @@ import com.google.gson.GsonBuilder;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.plugin.policyengine.*;
+import org.apache.ranger.plugin.policyevaluator.RangerPolicyEvaluator;
 import org.apache.ranger.plugin.util.ServicePolicies;
 
 import java.io.InputStream;
@@ -31,15 +32,20 @@ import java.io.InputStreamReader;
 import java.io.Reader;
 import java.net.URL;
 import java.nio.charset.Charset;
+import java.util.concurrent.atomic.AtomicLong;
 
 public class PerfTestEngine {
 	static final Log LOG      = LogFactory.getLog(PerfTestEngine.class);
 
+	static private final long POLICY_ENGINE_REORDER_AFTER_PROCESSING_REQUESTS_COUNT = 100;
 	private final URL servicePoliciesFileURL;
+	private final boolean isDynamicReorderingEnabled;
 	private RangerPolicyEngine policyEvaluationEngine;
+	private AtomicLong requestCount = new AtomicLong();
 
-	public PerfTestEngine(final URL servicePoliciesFileURL) {
+	public PerfTestEngine(final URL servicePoliciesFileURL, boolean isDynamicReorderingEnabled) {
 		this.servicePoliciesFileURL = servicePoliciesFileURL;
+		this.isDynamicReorderingEnabled = isDynamicReorderingEnabled;
 	}
 
 	public boolean init() {
@@ -66,9 +72,12 @@ public class PerfTestEngine {
 
 			RangerPolicyEngineOptions engineOptions = new RangerPolicyEngineOptions();
 			engineOptions.disableTagPolicyEvaluation = false;
+			engineOptions.evaluatorType = RangerPolicyEvaluator.EVALUATOR_TYPE_OPTIMIZED;
 
 			policyEvaluationEngine = new RangerPolicyEngineImpl("perf-test", servicePolicies, engineOptions);
 
+			requestCount.set(0L);
+
 			ret = true;
 
 		} catch (Exception excp) {
@@ -101,6 +110,12 @@ public class PerfTestEngine {
 
 		if (policyEvaluationEngine != null) {
 
+			long processedRequestCount = requestCount.getAndIncrement();
+
+			if (isDynamicReorderingEnabled && (processedRequestCount % POLICY_ENGINE_REORDER_AFTER_PROCESSING_REQUESTS_COUNT) == 0) {
+				policyEvaluationEngine.reorderPolicyEvaluators();
+			}
+
 			policyEvaluationEngine.preProcess(request);
 
 			ret = policyEvaluationEngine.isAccessAllowed(request, null);

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
index f30cbd7..321ee69 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/PerfTestOptions.java
@@ -26,17 +26,19 @@ public class PerfTestOptions {
 	private final URL servicePoliciesFileURL;
 	private final URL[] requestFileURLs;
 	private final URL statCollectionFileURL;
+	private final boolean isDynamicReorderingEnabled;
 
 
 	private final int concurrentClientCount;
 	private final int iterationsCount;
 
-	PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount) {
+	PerfTestOptions(URL servicePoliciesFileURL, URL[] requestFileURLs, URL statCollectionFileURL, int concurrentClientCount, int iterationsCount, boolean isDynamicReorderingEnabled) {
 		this.servicePoliciesFileURL = servicePoliciesFileURL;
 		this.requestFileURLs = requestFileURLs;
 		this.statCollectionFileURL = statCollectionFileURL;
 		this.iterationsCount = iterationsCount;
 		this.concurrentClientCount = concurrentClientCount;
+		this.isDynamicReorderingEnabled = isDynamicReorderingEnabled;
 	}
 
 	public URL getServicePoliciesFileURL() {
@@ -57,4 +59,7 @@ public class PerfTestOptions {
 
 	public int getIterationsCount() {
 		return iterationsCount;
-	}}
+	}
+
+	public boolean getIsDynamicReorderingEnabled() { return isDynamicReorderingEnabled; }
+}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
----------------------------------------------------------------------
diff --git a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
index e752c62..50432f7 100644
--- a/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
+++ b/ranger-tools/src/main/java/org/apache/ranger/policyengine/RangerPolicyenginePerfTester.java
@@ -54,7 +54,7 @@ public class RangerPolicyenginePerfTester {
 
             URL servicePoliciesFileURL = perfTestOptions.getServicePoliciesFileURL();
 
-            PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL);
+            PerfTestEngine perfTestEngine = new PerfTestEngine(servicePoliciesFileURL, perfTestOptions.getIsDynamicReorderingEnabled());
             if (!perfTestEngine.init()) {
                 LOG.error("Error initializing test data. Existing...");
                 System.exit(1);
@@ -63,6 +63,22 @@ public class RangerPolicyenginePerfTester {
             URL[] requestFileURLs = perfTestOptions.getRequestFileURLs();
             int requestFilesCount = requestFileURLs.length;
 
+            // warm-up policy engine
+            LOG.error("Warming up..");
+            try {
+                for(URL requestFileURL : requestFileURLs) {
+                    PerfTestClient perfTestClient = new PerfTestClient(perfTestEngine, 0, requestFileURL, 1);
+
+                    perfTestClient.init();
+                    perfTestClient.run();
+                }
+            } catch(Throwable t) {
+                LOG.error("Error during warmup", t);
+            }
+            LOG.error("Warmed up!");
+
+            PerfDataRecorder.clearStatistics();
+
             int clientsCount = perfTestOptions.getConcurrentClientCount();
             List<PerfTestClient> perfTestClients = new ArrayList<PerfTestClient>(clientsCount);
 
@@ -89,12 +105,9 @@ public class RangerPolicyenginePerfTester {
             long totalMemory = runtime.totalMemory();
             long freeMemory = runtime.freeMemory();
 
-            LOG.info("Initial Memory Statistics:");
-            LOG.info("\t\tMaximum memory available for the process:\t" + runtime.maxMemory());
-            LOG.info("\t\tInitial In-Use memory:\t\t\t\t" + (totalMemory-freeMemory));
-            LOG.info("\t\tInitial Free memory:\t\t\t\t" + freeMemory);
-            LOG.info("\n\n");
+            LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);
 
+            LOG.info("Starting " + perfTestClients.size() + " clients..");
             for (PerfTestClient client : perfTestClients) {
                 try {
                     client.start();
@@ -102,13 +115,13 @@ public class RangerPolicyenginePerfTester {
                     LOG.error("Error in starting client: " + client.getName(), t);
                 }
             }
+            LOG.info("Started " + perfTestClients.size() + " clients");
 
             LOG.info("Waiting for " + perfTestClients.size() + " clients to finish up");
 
             for (PerfTestClient client : perfTestClients) {
                 while (client.isAlive()) {
                     try {
-                        LOG.info("Waiting for " + client.getName() + " to finish up.");
                         client.join(1000);
 
                         runtime.gc();
@@ -116,10 +129,7 @@ public class RangerPolicyenginePerfTester {
                         totalMemory = runtime.totalMemory();
                         freeMemory = runtime.freeMemory();
 
-                        LOG.info("Memory Statistics:");
-                        LOG.info("\t\tCurrent In-Use memory:\t\t" + (totalMemory-freeMemory));
-                        LOG.info("\t\tCurrent Free memory:\t\t" + freeMemory);
-                        LOG.info("\n\n");
+                        LOG.info("Memory stats: max-available=:" + runtime.maxMemory() + "; in-use=" + (totalMemory-freeMemory) + "; free=" + freeMemory);
 
                     } catch (InterruptedException interruptedException) {
                         LOG.error("PerfTestClient.join() was interrupted");

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/ranger-tools/src/test/resources/testdata/test_modules.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/src/test/resources/testdata/test_modules.txt b/ranger-tools/src/test/resources/testdata/test_modules.txt
index 8eb5746..a355ec8 100644
--- a/ranger-tools/src/test/resources/testdata/test_modules.txt
+++ b/ranger-tools/src/test/resources/testdata/test_modules.txt
@@ -21,4 +21,5 @@ RangerTagEnricher.setServiceTags
 RangerPolicyEngine.init
 RangerPolicyEngine.preProcess
 RangerPolicyEngine.isAccessAllowedNoAudit
-
+RangerPolicyEngine.reorderPolicyEvaluators
+RangerPolicyEngine.usage

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/efe49a88/ranger-tools/testdata/test_modules.txt
----------------------------------------------------------------------
diff --git a/ranger-tools/testdata/test_modules.txt b/ranger-tools/testdata/test_modules.txt
index f317aaf..f3a2f82 100644
--- a/ranger-tools/testdata/test_modules.txt
+++ b/ranger-tools/testdata/test_modules.txt
@@ -20,6 +20,8 @@
 PolicyRefresher.loadPolicy
 RangerPolicyEngine.init
 RangerPolicyEngine.cleanUp
+RangerPolicyEngine.reorderPolicyEvaluators
+RangerPolicyEngine.usage
 RangerContextEnricher.init
 RangerPolicyEvaluator.init
 RangerPolicyItemEvaluator.init
@@ -58,4 +60,4 @@ ServiceREST.getPolicy
 ServiceREST.getPolicies
 ServiceREST.countPolicies
 ServiceREST.getServicePolicies
-ServiceREST.getServicePoliciesIfUpdated
\ No newline at end of file
+ServiceREST.getServicePoliciesIfUpdated


Mime
View raw message