Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 8B61B200BC4 for ; Sat, 5 Nov 2016 07:19:57 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 857B2160B07; Sat, 5 Nov 2016 06:19:57 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 168D2160AFE for ; Sat, 5 Nov 2016 07:19:54 +0100 (CET) Received: (qmail 84813 invoked by uid 500); 5 Nov 2016 06:19:54 -0000 Mailing-List: contact commits-help@ranger.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ranger.incubator.apache.org Delivered-To: mailing list commits@ranger.incubator.apache.org Received: (qmail 84802 invoked by uid 99); 5 Nov 2016 06:19:54 -0000 Received: from pnap-us-west-generic-nat.apache.org (HELO spamd1-us-west.apache.org) (209.188.14.142) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 05 Nov 2016 06:19:54 +0000 Received: from localhost (localhost [127.0.0.1]) by spamd1-us-west.apache.org (ASF Mail Server at spamd1-us-west.apache.org) with ESMTP id 6A287C1270 for ; Sat, 5 Nov 2016 06:19:53 +0000 (UTC) X-Virus-Scanned: Debian amavisd-new at spamd1-us-west.apache.org X-Spam-Flag: NO X-Spam-Score: -6.218 X-Spam-Level: X-Spam-Status: No, score=-6.218 tagged_above=-999 required=6.31 tests=[KAM_ASCII_DIVIDERS=0.8, KAM_LAZY_DOMAIN_SECURITY=1, RCVD_IN_DNSWL_HI=-5, RCVD_IN_MSPIKE_H3=-0.01, RCVD_IN_MSPIKE_WL=-0.01, RP_MATCHES_RCVD=-2.999, URIBL_BLOCKED=0.001] autolearn=disabled Received: from mx1-lw-eu.apache.org ([10.40.0.8]) by localhost (spamd1-us-west.apache.org [10.40.0.7]) (amavisd-new, port 10024) with ESMTP id uLJ1HJSvzXT1 for ; Sat, 5 Nov 2016 06:19:47 +0000 (UTC) Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx1-lw-eu.apache.org (ASF Mail Server at mx1-lw-eu.apache.org) with SMTP id 9D1805F3F4 for ; Sat, 5 Nov 2016 06:19:44 +0000 (UTC) Received: (qmail 84737 invoked by uid 99); 5 Nov 2016 06:19:43 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 05 Nov 2016 06:19:43 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 83524E0C0A; Sat, 5 Nov 2016 06:19:43 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: madhan@apache.org To: commits@ranger.incubator.apache.org Date: Sat, 05 Nov 2016 06:19:44 -0000 Message-Id: <9e261259608d444fb77d6decea73ff14@git.apache.org> In-Reply-To: <5b608c8a3fc34ef4b44756dd19d654c8@git.apache.org> References: <5b608c8a3fc34ef4b44756dd19d654c8@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [2/2] incubator-ranger git commit: RANGER-1197: track downloaded and active policy/tag versions in plugins archived-at: Sat, 05 Nov 2016 06:19:57 -0000 RANGER-1197: track downloaded and active policy/tag versions in plugins Signed-off-by: Madhan Neethiraj Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/f4a20e0b Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/f4a20e0b Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/f4a20e0b Branch: refs/heads/master Commit: f4a20e0b9e259a436cb27cb48988076eee4f443a Parents: 804753e Author: Abhay Kulkarni Authored: Mon Aug 1 15:00:35 2016 -0700 Committer: Madhan Neethiraj Committed: Fri Nov 4 20:49:19 2016 -0700 ---------------------------------------------------------------------- .../ranger/admin/client/RangerAdminClient.java | 4 +- .../admin/client/RangerAdminRESTClient.java | 27 +- .../RangerAdminTagRetriever.java | 4 +- .../RangerFileBasedTagRetriever.java | 6 +- .../contextenricher/RangerTagEnricher.java | 17 +- .../RangerTagFileStoreRetriever.java | 3 +- .../contextenricher/RangerTagRetriever.java | 2 +- .../ranger/plugin/model/RangerPluginInfo.java | 261 ++++++++++++++++++ .../ranger/plugin/util/PolicyRefresher.java | 16 +- .../ranger/plugin/util/RangerRESTUtils.java | 34 +++ .../apache/ranger/plugin/util/SearchFilter.java | 6 + .../hbase/RangerAdminClientImpl.java | 4 +- .../services/hdfs/RangerAdminClientImpl.java | 4 +- .../services/hive/RangerAdminClientImpl.java | 4 +- .../client/RangerAdminJersey2RESTClient.java | 10 +- .../kafka/authorizer/RangerAdminClientImpl.java | 4 +- .../kms/authorizer/RangerAdminClientImpl.java | 6 +- .../025-create-schema-for-plugin-info.sql | 33 +++ .../025-create-schema-for-plugin-info.sql | 35 +++ .../java/org/apache/ranger/biz/AssetMgr.java | 267 +++++++++++++++---- .../apache/ranger/common/RangerSearchUtil.java | 5 + ...RangerTransactionSynchronizationAdapter.java | 130 +++++++++ .../apache/ranger/db/RangerDaoManagerBase.java | 7 + .../org/apache/ranger/db/XXPluginInfoDao.java | 107 ++++++++ .../org/apache/ranger/entity/XXPluginInfo.java | 223 ++++++++++++++++ .../java/org/apache/ranger/rest/AssetREST.java | 2 +- .../org/apache/ranger/rest/PublicAPIsv2.java | 17 ++ .../org/apache/ranger/rest/ServiceREST.java | 92 +++++-- .../java/org/apache/ranger/rest/TagREST.java | 55 +++- .../apache/ranger/rest/TagRESTConstants.java | 1 + .../ranger/security/context/RangerAPIList.java | 3 + .../security/context/RangerAPIMapping.java | 2 + .../service/RangerPluginActivityLogger.java | 68 +++++ .../ranger/service/RangerPluginInfoService.java | 196 ++++++++++++++ .../service/RangerTransactionService.java | 105 ++++++++ .../ranger/view/RangerPluginInfoList.java | 72 +++++ .../resources/META-INF/jpa_named_queries.xml | 25 ++ .../src/main/resources/META-INF/persistence.xml | 2 + .../org/apache/ranger/rest/TestServiceREST.java | 2 +- .../org/apache/ranger/rest/TestTagREST.java | 31 ++- .../storm/RangerAdminClientImpl.java | 4 +- 41 files changed, 1771 insertions(+), 125 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java index 5ae9854..6755e15 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminClient.java @@ -30,13 +30,13 @@ import java.util.List; public interface RangerAdminClient { void init(String serviceName, String appId, String configPropertyPrefix); - ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion) throws Exception; + ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception; void grantAccess(GrantRevokeRequest request) throws Exception; void revokeAccess(GrantRevokeRequest request) throws Exception; - ServiceTags getServiceTagsIfUpdated(long lastKnownVersion) throws Exception; + ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception; List getTagTypes(String tagTypePattern) throws Exception; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java index 6ec44c4..646004a 100644 --- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java +++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java @@ -31,7 +31,11 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.ranger.admin.client.datatype.RESTResponse; import org.apache.ranger.audit.provider.MiscUtil; import org.apache.ranger.authorization.hadoop.config.RangerConfiguration; -import org.apache.ranger.plugin.util.*; +import org.apache.ranger.plugin.util.GrantRevokeRequest; +import org.apache.ranger.plugin.util.RangerRESTClient; +import org.apache.ranger.plugin.util.RangerRESTUtils; +import org.apache.ranger.plugin.util.ServicePolicies; +import org.apache.ranger.plugin.util.ServiceTags; import java.lang.reflect.ParameterizedType; import java.lang.reflect.Type; @@ -44,7 +48,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { private String serviceName = null; private String pluginId = null; private RangerRESTClient restClient = null; - private RangerRESTUtils restUtils = new RangerRESTUtils(); + private RangerRESTUtils restUtils = new RangerRESTUtils(); public RangerAdminRESTClient() { } @@ -82,9 +86,9 @@ public class RangerAdminRESTClient implements RangerAdminClient { } @Override - public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion) throws Exception { + public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ")"); + LOG.debug("==> RangerAdminRESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); } ServicePolicies ret = null; @@ -100,6 +104,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { public ClientResponse run() { WebResource secureWebResource = createWebResource(RangerRESTUtils.REST_URL_POLICY_GET_FOR_SECURE_SERVICE_IF_UPDATED + serviceName) .queryParam(RangerRESTUtils.REST_PARAM_LAST_KNOWN_POLICY_VERSION, Long.toString(lastKnownVersion)) + .queryParam(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)) .queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); return secureWebResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class); }; @@ -111,6 +116,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { } WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_POLICY_GET_FOR_SERVICE_IF_UPDATED + serviceName) .queryParam(RangerRESTUtils.REST_PARAM_LAST_KNOWN_POLICY_VERSION, Long.toString(lastKnownVersion)) + .queryParam(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)) .queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); response = webResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class); } @@ -125,7 +131,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerAdminRESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + "): " + ret); + LOG.debug("<== RangerAdminRESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + ret); } return ret; @@ -243,9 +249,9 @@ public class RangerAdminRESTClient implements RangerAdminClient { } @Override - public ServiceTags getServiceTagsIfUpdated(final long lastKnownVersion) throws Exception { + public ServiceTags getServiceTagsIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAdminRESTClient.getServiceTagsIfUpdated(" + lastKnownVersion + "): "); + LOG.debug("==> RangerAdminRESTClient.getServiceTagsIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): "); } ServiceTags ret = null; @@ -259,6 +265,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { public ClientResponse run() { WebResource secureWebResource = createWebResource(RangerRESTUtils.REST_URL_GET_SECURE_SERVICE_TAGS_IF_UPDATED + serviceName) .queryParam(RangerRESTUtils.LAST_KNOWN_TAG_VERSION_PARAM, Long.toString(lastKnownVersion)) + .queryParam(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)) .queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); return secureWebResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class); }; @@ -270,6 +277,7 @@ public class RangerAdminRESTClient implements RangerAdminClient { } else { webResource = createWebResource(RangerRESTUtils.REST_URL_GET_SERVICE_TAGS_IF_UPDATED + serviceName) .queryParam(RangerRESTUtils.LAST_KNOWN_TAG_VERSION_PARAM, Long.toString(lastKnownVersion)) + .queryParam(RangerRESTUtils.REST_PARAM_LAST_ACTIVATION_TIME, Long.toString(lastActivationTimeInMillis)) .queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId); response = webResource.accept(RangerRESTUtils.REST_MIME_TYPE_JSON).get(ClientResponse.class); } @@ -280,12 +288,13 @@ public class RangerAdminRESTClient implements RangerAdminClient { RESTResponse resp = RESTResponse.fromClientResponse(response); LOG.error("Error getting taggedResources. secureMode=" + isSecureMode + ", user=" + user + ", response=" + resp.toString() + ", serviceName=" + serviceName - + ", " + "lastKnownVersion=" + lastKnownVersion); + + ", " + "lastKnownVersion=" + lastKnownVersion + + ", " + "lastActivationTimeInMillis=" + lastActivationTimeInMillis); throw new Exception(resp.getMessage()); } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerAdminRESTClient.getServiceTagsIfUpdated(" + lastKnownVersion + "): "); + LOG.debug("<== RangerAdminRESTClient.getServiceTagsIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): "); } return ret; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java index 7c5b378..9c336bf 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java @@ -48,13 +48,13 @@ public class RangerAdminTagRetriever extends RangerTagRetriever { } @Override - public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException { + public ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeInMillis) throws InterruptedException { ServiceTags serviceTags = null; if (adminClient != null) { try { - serviceTags = adminClient.getServiceTagsIfUpdated(lastKnownVersion); + serviceTags = adminClient.getServiceTagsIfUpdated(lastKnownVersion, lastActivationTimeInMillis); } catch (InterruptedException interruptedException) { LOG.error("Tag-retriever thread was interrupted"); throw interruptedException; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java index db4c6e4..248aafa 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerFileBasedTagRetriever.java @@ -124,10 +124,10 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever { } @Override - public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException { + public ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeInMillis) throws InterruptedException { if (LOG.isDebugEnabled()) { - LOG.debug("==> retrieveTags(lastKnownVersion=" + lastKnownVersion + ", serviceTagsFilePath=" + serviceTagsFileName); + LOG.debug("==> retrieveTags(lastKnownVersion=" + lastKnownVersion + ", lastActivationTimeInMillis=" + lastActivationTimeInMillis + ", serviceTagsFilePath=" + serviceTagsFileName); } ServiceTags serviceTags = null; @@ -152,7 +152,7 @@ public class RangerFileBasedTagRetriever extends RangerTagRetriever { } if (LOG.isDebugEnabled()) { - LOG.debug("<== retrieveTags(lastKnownVersion=" + lastKnownVersion); + LOG.debug("<== retrieveTags(lastKnownVersion=" + lastKnownVersion + ", lastActivationTimeInMillis=" + lastActivationTimeInMillis); } return serviceTags; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java index 00e46ea..62e7c99 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagEnricher.java @@ -357,6 +357,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { private final RangerTagRetriever tagRetriever; private final RangerTagEnricher tagEnricher; private long lastKnownVersion = -1L; + private long lastActivationTimeInMillis = 0L; private final long pollingIntervalMs; private final String cacheFile; @@ -381,6 +382,14 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { } } + public long getLastActivationTimeInMillis() { + return lastActivationTimeInMillis; + } + + public void setLastActivationTimeInMillis(long lastActivationTimeInMillis) { + this.lastActivationTimeInMillis = lastActivationTimeInMillis; + } + @Override public void run() { @@ -423,7 +432,7 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { if (tagEnricher != null) { ServiceTags serviceTags = null; - serviceTags = tagRetriever.retrieveTags(lastKnownVersion); + serviceTags = tagRetriever.retrieveTags(lastKnownVersion, lastActivationTimeInMillis); if (serviceTags == null) { if (!hasProvidedTagsToReceiver) { @@ -435,9 +444,11 @@ public class RangerTagEnricher extends RangerAbstractContextEnricher { if (serviceTags != null) { tagEnricher.setServiceTags(serviceTags); - lastKnownVersion = serviceTags.getTagVersion() == null ? -1L : serviceTags.getTagVersion(); - LOG.info("RangerTagRefresher.populateTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion()); + LOG.info("RangerTagRefresher.populateTags() - Updated tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + + (serviceTags.getTagVersion() == null ? -1L : serviceTags.getTagVersion())); hasProvidedTagsToReceiver = true; + lastKnownVersion = serviceTags.getTagVersion() == null ? -1L : serviceTags.getTagVersion(); + setLastActivationTimeInMillis(System.currentTimeMillis()); } else { if (LOG.isDebugEnabled()) { LOG.debug("RangerTagRefresher.populateTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java index 1ee00d3..85cf420 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagFileStoreRetriever.java @@ -47,12 +47,13 @@ public class RangerTagFileStoreRetriever extends RangerTagRetriever { } @Override - public ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException { + public ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeInMillis) throws InterruptedException { ServiceTags serviceTags = null; if (tagStore != null) { try { + // Ignore lastActivationTimeInMillis for TagFileStore serviceTags = tagStore.getServiceTagsIfUpdated(serviceName, lastKnownVersion); } catch (InterruptedException interruptedException) { LOG.error("Tag-retriever thread was interrupted"); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java index 3ec1754..a5eeeaa 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagRetriever.java @@ -32,7 +32,7 @@ public abstract class RangerTagRetriever { public abstract void init(Map options); - public abstract ServiceTags retrieveTags(long lastKnownVersion) throws InterruptedException; + public abstract ServiceTags retrieveTags(long lastKnownVersion, long lastActivationTimeInMillis) throws InterruptedException; public String getServiceName() { return serviceName; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPluginInfo.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPluginInfo.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPluginInfo.java new file mode 100644 index 0000000..ee11c42 --- /dev/null +++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerPluginInfo.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.plugin.model; + +import org.apache.commons.lang.StringUtils; +import org.codehaus.jackson.annotate.JsonAutoDetect; +import org.codehaus.jackson.annotate.JsonIgnore; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +@JsonAutoDetect(fieldVisibility=JsonAutoDetect.Visibility.ANY) +@JsonSerialize(include=JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown=true) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerPluginInfo implements Serializable { + private static final long serialVersionUID = 1L; + + public static final int ENTITY_TYPE_POLICIES = 0; + public static final int ENTITY_TYPE_TAGS = 1; + + public static final String PLUGIN_INFO_POLICY_DOWNLOAD_TIME = "policyDownloadTime"; + public static final String PLUGIN_INFO_POLICY_DOWNLOADED_VERSION = "policyDownloadedVersion"; + public static final String PLUGIN_INFO_POLICY_ACTIVATION_TIME = "policyActivationTime"; + public static final String PLUGIN_INFO_POLICY_ACTIVE_VERSION = "policyActiveVersion"; + public static final String PLUGIN_INFO_TAG_DOWNLOAD_TIME = "tagDownloadTime"; + public static final String PLUGIN_INFO_TAG_DOWNLOADED_VERSION = "tagDownloadedVersion"; + public static final String PLUGIN_INFO_TAG_ACTIVATION_TIME = "tagActivationTime"; + public static final String PLUGIN_INFO_TAG_ACTIVE_VERSION = "tagActiveVersion"; + + private Long id = null; + private Date createTime = null; + private Date updateTime = null; + + private String serviceName; + private String hostName; + private String appType; + private String ipAddress; + private Map info; + + public RangerPluginInfo(Long id, Date createTime, Date updateTime, String serviceName, String appType, String hostName, String ipAddress, Map info) { + super(); + + setId(id); + setCreateTime(createTime); + setUpdateTime(updateTime); + setServiceName(serviceName); + setAppType(appType); + setHostName(hostName); + setIpAddress(ipAddress); + setInfo(info); + } + + public RangerPluginInfo() { + this(null, null, null, null, null, null, null, null); + } + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + public Date getUpdateTime() { + return updateTime; + } + + public void setUpdateTime(Date updateTime) { + this.updateTime = updateTime; + } + + public String getServiceName() { + return serviceName; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public String getHostName() { + return hostName; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public String getAppType() { + return appType; + } + + public void setAppType(String appType) { + this.appType = appType; + } + + public String getIpAddress() { + return ipAddress; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public Map getInfo() { + return info; + } + + public void setInfo(Map info) { + this.info = info == null ? new HashMap() : info; + } + + @JsonIgnore + public void setPolicyDownloadTime(Long policyDownloadTime) { + getInfo().put(PLUGIN_INFO_POLICY_DOWNLOAD_TIME, policyDownloadTime == null ? null : Long.toString(policyDownloadTime)); + } + + @JsonIgnore + public Long getPolicyDownloadTime() { + String downloadTimeString = getInfo().get(PLUGIN_INFO_POLICY_DOWNLOAD_TIME); + return StringUtils.isNotBlank(downloadTimeString) ? Long.valueOf(downloadTimeString) : null; + } + + @JsonIgnore + public void setPolicyDownloadedVersion(Long policyDownloadedVersion) { + getInfo().put(PLUGIN_INFO_POLICY_DOWNLOADED_VERSION, policyDownloadedVersion == null ? null : Long.toString(policyDownloadedVersion)); + } + + @JsonIgnore + public Long getPolicyDownloadedVersion() { + String downloadedVersionString = getInfo().get(PLUGIN_INFO_POLICY_DOWNLOADED_VERSION); + return StringUtils.isNotBlank(downloadedVersionString) ? Long.valueOf(downloadedVersionString) : null; + } + + @JsonIgnore + public void setPolicyActivationTime(Long policyActivationTime) { + getInfo().put(PLUGIN_INFO_POLICY_ACTIVATION_TIME, policyActivationTime == null ? null : Long.toString(policyActivationTime)); + } + + @JsonIgnore + public Long getPolicyActivationTime() { + String activationTimeString = getInfo().get(PLUGIN_INFO_POLICY_ACTIVATION_TIME); + return StringUtils.isNotBlank(activationTimeString) ? Long.valueOf(activationTimeString) : null; + } + + @JsonIgnore + public void setPolicyActiveVersion(Long policyActiveVersion) { + getInfo().put(PLUGIN_INFO_POLICY_ACTIVE_VERSION, policyActiveVersion == null ? null : Long.toString(policyActiveVersion)); + } + + @JsonIgnore + public Long getPolicyActiveVersion() { + String activeVersionString = getInfo().get(PLUGIN_INFO_POLICY_ACTIVE_VERSION); + return StringUtils.isNotBlank(activeVersionString) ? Long.valueOf(activeVersionString) : null; + } + + @JsonIgnore + public void setTagDownloadTime(Long tagDownloadTime) { + getInfo().put(PLUGIN_INFO_TAG_DOWNLOAD_TIME, tagDownloadTime == null ? null : Long.toString(tagDownloadTime)); + } + + @JsonIgnore + public Long getTagDownloadTime() { + String downloadTimeString = getInfo().get(PLUGIN_INFO_TAG_DOWNLOAD_TIME); + return StringUtils.isNotBlank(downloadTimeString) ? Long.valueOf(downloadTimeString) : null; + } + + @JsonIgnore + public void setTagDownloadedVersion(Long tagDownloadedVersion) { + getInfo().put(PLUGIN_INFO_TAG_DOWNLOADED_VERSION, tagDownloadedVersion == null ? null : Long.toString(tagDownloadedVersion)); + } + + @JsonIgnore + public Long getTagDownloadedVersion() { + String downloadedVersion = getInfo().get(PLUGIN_INFO_TAG_DOWNLOADED_VERSION); + return StringUtils.isNotBlank(downloadedVersion) ? Long.valueOf(downloadedVersion) : null; + } + + @JsonIgnore + public void setTagActivationTime(Long tagActivationTime) { + getInfo().put(PLUGIN_INFO_TAG_ACTIVATION_TIME, tagActivationTime == null ? null : Long.toString(tagActivationTime)); + } + + @JsonIgnore + public Long getTagActivationTime() { + String activationTimeString = getInfo().get(PLUGIN_INFO_TAG_ACTIVATION_TIME); + return StringUtils.isNotBlank(activationTimeString) ? Long.valueOf(activationTimeString) : null; + } + + @JsonIgnore + public void setTagActiveVersion(Long tagActiveVersion) { + getInfo().put(PLUGIN_INFO_TAG_ACTIVE_VERSION, tagActiveVersion == null ? null : Long.toString(tagActiveVersion)); + } + + @JsonIgnore + public Long getTagActiveVersion() { + String activeVersionString = getInfo().get(PLUGIN_INFO_TAG_ACTIVE_VERSION); + return StringUtils.isNotBlank(activeVersionString) ? Long.valueOf(activeVersionString) : null; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPluginInfo={"); + + sb.append("id={").append(id).append("} "); + sb.append("createTime={").append(createTime).append("} "); + sb.append("updateTime={").append(updateTime).append("} "); + sb.append("serviceName={").append(serviceName).append("} "); + sb.append("hostName={").append(hostName).append("} "); + sb.append("appType={").append(appType).append("} "); + sb.append("ipAddress={").append(ipAddress).append("} "); + sb.append("info={").append(info).append("} "); + + sb.append(" }"); + + return sb; + } +} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java index 014e866..91c24c6 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/PolicyRefresher.java @@ -49,7 +49,8 @@ public class PolicyRefresher extends Thread { private final Gson gson; private long pollingIntervalMs = 30 * 1000; - private long lastKnownVersion = -1; + private long lastKnownVersion = -1L; + private long lastActivationTimeInMillis = 0L; private boolean policiesSetInPlugin = false; @@ -130,6 +131,13 @@ public class PolicyRefresher extends Thread { this.pollingIntervalMs = pollingIntervalMilliSeconds; } + public long getLastActivationTimeInMillis() { + return lastActivationTimeInMillis; + } + + public void setLastActivationTimeInMillis(long lastActivationTimeInMillis) { + this.lastActivationTimeInMillis = lastActivationTimeInMillis; + } public void startRefresher() { @@ -207,6 +215,8 @@ public class PolicyRefresher extends Thread { if (svcPolicies != null) { plugIn.setPolicies(svcPolicies); policiesSetInPlugin = true; + setLastActivationTimeInMillis(System.currentTimeMillis()); + lastKnownVersion = svcPolicies.getPolicyVersion(); } if(LOG.isDebugEnabled()) { @@ -229,7 +239,7 @@ public class PolicyRefresher extends Thread { } try { - svcPolicies = rangerAdmin.getServicePoliciesIfUpdated(lastKnownVersion); + svcPolicies = rangerAdmin.getServicePoliciesIfUpdated(lastKnownVersion, lastActivationTimeInMillis); boolean isUpdated = svcPolicies != null; @@ -244,8 +254,6 @@ public class PolicyRefresher extends Thread { LOG.info("PolicyRefresher(serviceName=" + serviceName + "): found updated version. lastKnownVersion=" + lastKnownVersion + "; newVersion=" + newVersion); - lastKnownVersion = newVersion; - } else { if(LOG.isDebugEnabled()) { LOG.debug("PolicyRefresher(serviceName=" + serviceName + ").run(): no update found. lastKnownVersion=" + lastKnownVersion); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java index ed674ee..fa81f0b 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java @@ -56,6 +56,7 @@ public class RangerRESTUtils { public static final String REST_MIME_TYPE_JSON = "application/json"; public static final String REST_PARAM_LAST_KNOWN_POLICY_VERSION = "lastKnownVersion"; + public static final String REST_PARAM_LAST_ACTIVATION_TIME = "lastActivationTime"; public static final String REST_PARAM_PLUGIN_ID = "pluginId"; private static final int MAX_PLUGIN_ID_LEN = 255; @@ -130,4 +131,37 @@ public class RangerRESTUtils { return ret ; } + + public String getHostnameFromPluginId(String pluginId, String serviceName) { + String ret = ""; + + if (StringUtils.isNotBlank(pluginId)) { + int lastIndex; + String[] parts = pluginId.split("@"); + int index = parts.length > 1 ? 1 : 0; + if (StringUtils.isNotBlank(serviceName)) { + lastIndex = StringUtils.lastIndexOf(parts[index], serviceName); + if (lastIndex > 1) { + ret = parts[index].substring(0, lastIndex-1); + } + } else { + lastIndex = StringUtils.lastIndexOf(parts[index], "-"); + if (lastIndex > 0) { + ret = parts[index].substring(0, lastIndex); + } + } + } + + return ret; + } + public String getAppIdFromPluginId(String pluginId) { + String ret = "**Unknown**"; + + if (StringUtils.isNotBlank(pluginId)) { + String[] parts = pluginId.split("@"); + ret = parts[0]; + } + + return ret; + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java index 49a48cd..73ea6e9 100644 --- a/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/SearchFilter.java @@ -65,6 +65,12 @@ public class SearchFilter { public static final String TAG_MAP_GUID = "tagResourceMapGuid"; // search public static final String SERVICE_NAME_PARTIAL = "serviceNamePartial"; + + public static final String PLUGIN_HOST_NAME = "pluginHostName"; + public static final String PLUGIN_APP_TYPE = "pluginAppType"; + public static final String PLUGIN_ENTITY_TYPE = "pluginEntityType"; + public static final String PLUGIN_IP_ADDRESS = "pluginIpAddress"; + private Map params = null; private int startIndex = 0; private int maxRows = Integer.MAX_VALUE; http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/RangerAdminClientImpl.java ---------------------------------------------------------------------- diff --git a/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/RangerAdminClientImpl.java b/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/RangerAdminClientImpl.java index 3269fbb..e5c4113 100644 --- a/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/RangerAdminClientImpl.java +++ b/hbase-agent/src/test/java/org/apache/ranger/authorization/hbase/RangerAdminClientImpl.java @@ -50,7 +50,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { this.gson = gson; } - public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion) throws Exception { + public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { String basedir = System.getProperty("basedir"); if (basedir == null) { @@ -71,7 +71,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { } - public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion) throws Exception { + public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { return null; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerAdminClientImpl.java ---------------------------------------------------------------------- diff --git a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerAdminClientImpl.java b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerAdminClientImpl.java index 5c612e9..99d3f78 100644 --- a/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerAdminClientImpl.java +++ b/hdfs-agent/src/test/java/org/apache/ranger/services/hdfs/RangerAdminClientImpl.java @@ -50,7 +50,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { this.gson = gson; } - public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion) throws Exception { + public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { String basedir = System.getProperty("basedir"); if (basedir == null) { @@ -71,7 +71,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { } - public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion) throws Exception { + public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { return null; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/hive-agent/src/test/java/org/apache/ranger/services/hive/RangerAdminClientImpl.java ---------------------------------------------------------------------- diff --git a/hive-agent/src/test/java/org/apache/ranger/services/hive/RangerAdminClientImpl.java b/hive-agent/src/test/java/org/apache/ranger/services/hive/RangerAdminClientImpl.java index 43770c2..8b48dd8 100644 --- a/hive-agent/src/test/java/org/apache/ranger/services/hive/RangerAdminClientImpl.java +++ b/hive-agent/src/test/java/org/apache/ranger/services/hive/RangerAdminClientImpl.java @@ -50,7 +50,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { this.gson = gson; } - public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion) throws Exception { + public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { String basedir = System.getProperty("basedir"); if (basedir == null) { @@ -71,7 +71,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { } - public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion) throws Exception { + public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { return null; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java ---------------------------------------------------------------------- diff --git a/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java index aeff332..6c0b3e9 100644 --- a/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java +++ b/knox-agent/src/main/java/org/apache/ranger/admin/client/RangerAdminJersey2RESTClient.java @@ -91,9 +91,9 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient { } @Override - public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion) throws Exception { + public ServicePolicies getServicePoliciesIfUpdated(final long lastKnownVersion, final long lastActivationTimeInMillis) throws Exception { if(LOG.isDebugEnabled()) { - LOG.debug("==> RangerAdminJersey2RESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ")"); + LOG.debug("==> RangerAdminJersey2RESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + ")"); } UserGroupInformation user = MiscUtil.getUGILoginUser(); @@ -162,11 +162,11 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient { } if(LOG.isDebugEnabled()) { - LOG.debug("<== RangerAdminJersey2RESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + "): " + servicePolicies); + LOG.debug("<== RangerAdminJersey2RESTClient.getServicePoliciesIfUpdated(" + lastKnownVersion + ", " + lastActivationTimeInMillis + "): " + servicePolicies); } return servicePolicies; } catch (Exception ex) { - LOG.error("Failed getting policies from server. url=" + url + ", pluginId=" + _pluginId + ", lastKnownVersion=" + lastKnownVersion ); + LOG.error("Failed getting policies from server. url=" + url + ", pluginId=" + _pluginId + ", lastKnownVersion=" + lastKnownVersion + ", " + lastActivationTimeInMillis); throw ex; } } @@ -242,7 +242,7 @@ public class RangerAdminJersey2RESTClient implements RangerAdminClient { } @Override - public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion) throws Exception { + public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { throw new Exception("RangerAdminjersey2RESTClient.getServiceTagsIfUpdated() -- *** NOT IMPLEMENTED *** "); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/RangerAdminClientImpl.java ---------------------------------------------------------------------- diff --git a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/RangerAdminClientImpl.java b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/RangerAdminClientImpl.java index c08a6df..26dc151 100644 --- a/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/RangerAdminClientImpl.java +++ b/plugin-kafka/src/test/java/org/apache/ranger/authorization/kafka/authorizer/RangerAdminClientImpl.java @@ -50,7 +50,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { this.gson = gson; } - public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion) throws Exception { + public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { String basedir = System.getProperty("basedir"); if (basedir == null) { @@ -71,7 +71,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { } - public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion) throws Exception { + public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { return null; } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/plugin-kms/src/test/java/org/apache/ranger/authorization/kms/authorizer/RangerAdminClientImpl.java ---------------------------------------------------------------------- diff --git a/plugin-kms/src/test/java/org/apache/ranger/authorization/kms/authorizer/RangerAdminClientImpl.java b/plugin-kms/src/test/java/org/apache/ranger/authorization/kms/authorizer/RangerAdminClientImpl.java index e889447..6196727 100644 --- a/plugin-kms/src/test/java/org/apache/ranger/authorization/kms/authorizer/RangerAdminClientImpl.java +++ b/plugin-kms/src/test/java/org/apache/ranger/authorization/kms/authorizer/RangerAdminClientImpl.java @@ -50,7 +50,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { this.gson = gson; } - public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion) throws Exception { + public ServicePolicies getServicePoliciesIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { String basedir = System.getProperty("basedir"); if (basedir == null) { @@ -71,7 +71,7 @@ public class RangerAdminClientImpl implements RangerAdminClient { } - public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion) throws Exception { + public ServiceTags getServiceTagsIfUpdated(long lastKnownVersion, long lastActivationTimeInMillis) throws Exception { return null; } @@ -81,4 +81,4 @@ public class RangerAdminClientImpl implements RangerAdminClient { } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/db/mysql/patches/025-create-schema-for-plugin-info.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/mysql/patches/025-create-schema-for-plugin-info.sql b/security-admin/db/mysql/patches/025-create-schema-for-plugin-info.sql new file mode 100644 index 0000000..732f2cf --- /dev/null +++ b/security-admin/db/mysql/patches/025-create-schema-for-plugin-info.sql @@ -0,0 +1,33 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +DROP TABLE IF EXISTS `x_plugin_info`; + + +CREATE TABLE x_plugin_info ( +`id` bigint(20) NOT NULL AUTO_INCREMENT, +`create_time` datetime NOT NULL, +`update_time` datetime NOT NULL, +`service_name` varchar(255) NOT NULL, +`app_type` varchar(128) NOT NULL, +`host_name` varchar(64) NOT NULL, +`ip_address` varchar(64) NOT NULL, +`info` varchar(1024) NOT NULL, + PRIMARY KEY (`id`), + CREATE UNIQUE INDEX x_plugin_info_IDX_unique ON x_plugin_info(service_name, host_name, app_type), + CREATE INDEX x_plugin_info_IDX_service_name ON x_plugin_info(service_name), + CREATE INDEX x_plugin_info_IDX_host_name ON x_plugin_info(host_name) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/db/postgres/patches/025-create-schema-for-plugin-info.sql ---------------------------------------------------------------------- diff --git a/security-admin/db/postgres/patches/025-create-schema-for-plugin-info.sql b/security-admin/db/postgres/patches/025-create-schema-for-plugin-info.sql new file mode 100644 index 0000000..1b12915 --- /dev/null +++ b/security-admin/db/postgres/patches/025-create-schema-for-plugin-info.sql @@ -0,0 +1,35 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + +DROP TABLE IF EXISTS x_plugin_info; +DROP SEQUENCE IF EXISTS x_plugin_info_seq; + +CREATE SEQUENCE x_plugin_info_seq; + +CREATE TABLE x_plugin_info ( +id BIGINT DEFAULT nextval('x_plugin_info_seq'::regclass), +create_time TIMESTAMP NOT NULL, +update_time TIMESTAMP NOT NULL, +service_name varchar(255) NOT NULL, +app_type varchar(128) NOT NULL, +host_name varchar(64) NOT NULL, +ip_address varchar(64) NOT NULL, +info varchar(1024) NOT NULL, +primary key (id) +); +CREATE UNIQUE INDEX x_plugin_info_IDX_unique ON x_plugin_info(service_name, host_name, app_type); +CREATE INDEX x_plugin_info_IDX_service_name ON x_plugin_info(service_name); +CREATE INDEX x_plugin_info_IDX_host_name ON x_plugin_info(host_name); + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java index 931356e..3df1aba 100644 --- a/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java +++ b/security-admin/src/main/java/org/apache/ranger/biz/AssetMgr.java @@ -23,6 +23,7 @@ import java.io.File; import java.io.IOException; import java.security.cert.X509Certificate; import java.util.ArrayList; +import java.util.Date; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -32,8 +33,10 @@ import java.util.Set; import javax.naming.InvalidNameException; import javax.naming.ldap.LdapName; import javax.naming.ldap.Rdn; +import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.ranger.common.AppConstants; import org.apache.ranger.common.DateUtil; @@ -45,10 +48,15 @@ import org.apache.ranger.common.SearchCriteria; import org.apache.ranger.common.StringUtil; import org.apache.ranger.db.RangerDaoManager; import org.apache.ranger.entity.XXPermMap; +import org.apache.ranger.entity.XXPluginInfo; import org.apache.ranger.entity.XXPolicyExportAudit; import org.apache.ranger.entity.XXPortalUser; import org.apache.ranger.entity.XXTrxLog; import org.apache.ranger.entity.XXUser; +import org.apache.ranger.plugin.model.RangerPluginInfo; +import org.apache.ranger.plugin.util.RangerRESTUtils; +import org.apache.ranger.service.RangerPluginActivityLogger; +import org.apache.ranger.service.RangerPluginInfoService; import org.apache.ranger.service.XAccessAuditService; import org.apache.ranger.service.XAuditMapService; import org.apache.ranger.service.XGroupService; @@ -58,25 +66,26 @@ import org.apache.ranger.service.XTrxLogService; import org.apache.ranger.service.XUserService; import org.apache.ranger.solr.SolrAccessAuditsService; import org.apache.ranger.util.RestUtil; -import org.apache.ranger.view.*; +import org.apache.ranger.view.VXAccessAuditList; +import org.apache.ranger.view.VXAsset; +import org.apache.ranger.view.VXAuditMap; +import org.apache.ranger.view.VXPermMap; +import org.apache.ranger.view.VXPolicyExportAuditList; +import org.apache.ranger.view.VXResource; +import org.apache.ranger.view.VXTrxLog; +import org.apache.ranger.view.VXTrxLogList; +import org.apache.ranger.view.VXUser; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.map.JsonMappingException; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionDefinition; -import org.springframework.transaction.TransactionStatus; -import org.springframework.transaction.support.TransactionCallback; -import org.springframework.transaction.support.TransactionTemplate; @Component public class AssetMgr extends AssetMgrBase { - - + @Autowired XPermMapService xPermMapService; - + @Autowired XAuditMapService xAuditMapService; @@ -114,14 +123,15 @@ public class AssetMgr extends AssetMgrBase { SolrAccessAuditsService solrAccessAuditsService; @Autowired - @Qualifier(value = "transactionManager") - PlatformTransactionManager txManager; - - @Autowired XPolicyService xPolicyService; - - static Logger logger = Logger.getLogger(AssetMgr.class); + @Autowired + RangerPluginActivityLogger activityLogger; + + @Autowired + RangerPluginInfoService pluginInfoService; + + static Logger logger = Logger.getLogger(AssetMgr.class); public File getXResourceFile(Long id, String fileType) { VXResource xResource = xResourceService.readResource(id); @@ -171,8 +181,8 @@ public class AssetMgr extends AssetMgrBase { } public String getLatestRepoPolicy(VXAsset xAsset, List xResourceList, Long updatedTime, - X509Certificate[] certchain, boolean httpEnabled, String epoch, - String ipAddress, boolean isSecure, String count, String agentId) { + X509Certificate[] certchain, boolean httpEnabled, String epoch, + String ipAddress, boolean isSecure, String count, String agentId) { if(xAsset==null){ logger.error("Requested repository not found"); throw restErrorUtil.createRESTException("No Data Found.", @@ -631,26 +641,193 @@ public class AssetMgr extends AssetMgrBase { public XXPolicyExportAudit createPolicyAudit( final XXPolicyExportAudit xXPolicyExportAudit) { - TransactionTemplate txTemplate = new TransactionTemplate(txManager); - txTemplate - .setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRED); - XXPolicyExportAudit policyExportAudit = (XXPolicyExportAudit) txTemplate - .execute(new TransactionCallback() { - public Object doInTransaction(TransactionStatus status) { - if (xXPolicyExportAudit.getHttpRetCode() == HttpServletResponse.SC_NOT_MODIFIED) { - boolean logNotModified = PropertiesUtil.getBooleanProperty("ranger.log.SC_NOT_MODIFIED", false); - if (!logNotModified) { - logger.debug("Not logging HttpServletResponse." - + "SC_NOT_MODIFIED, to enable, update " - + ": ranger.log.SC_NOT_MODIFIED"); - return null; - } + + XXPolicyExportAudit ret = null; + if (xXPolicyExportAudit.getHttpRetCode() == HttpServletResponse.SC_NOT_MODIFIED) { + boolean logNotModified = PropertiesUtil.getBooleanProperty("ranger.log.SC_NOT_MODIFIED", false); + if (!logNotModified) { + logger.debug("Not logging HttpServletResponse." + + "SC_NOT_MODIFIED, to enable, update " + + ": ranger.log.SC_NOT_MODIFIED"); + } else { + // Create PolicyExportAudit record after transaction is completed. If it is created in-line here + // then the TransactionManager will roll-back the changes because the HTTP return code is + // HttpServletResponse.SC_NOT_MODIFIED + Runnable commitWork = new Runnable() { + @Override + public void run() { + rangerDaoManager.getXXPolicyExportAudit().create(xXPolicyExportAudit); + + } + }; + activityLogger.commitAfterTransactionComplete(commitWork); + } + } else { + ret = rangerDaoManager.getXXPolicyExportAudit().create(xXPolicyExportAudit); + } + + return ret; + } + + public void createPluginInfo(String serviceName, String pluginId, HttpServletRequest request, int entityType, long downloadedVersion, long lastKnownVersion, long lastActivationTime, int httpCode) { + RangerRESTUtils restUtils = new RangerRESTUtils(); + + final String ipAddress = request != null ? request.getRemoteAddr() : null; + final String appType = restUtils.getAppIdFromPluginId(pluginId); + + String tmpHostName = null; + if (StringUtils.isNotBlank(pluginId)) { + tmpHostName = restUtils.getHostnameFromPluginId(pluginId, serviceName); + } + if (StringUtils.isBlank(tmpHostName) && request != null) { + tmpHostName = request.getRemoteHost(); + } + + final String hostName = (StringUtils.isBlank(tmpHostName)) ? ipAddress : tmpHostName; + + RangerPluginInfo pluginSvcVersionInfo = new RangerPluginInfo(); + + pluginSvcVersionInfo.setServiceName(serviceName); + pluginSvcVersionInfo.setAppType(appType); + pluginSvcVersionInfo.setHostName(hostName); + pluginSvcVersionInfo.setIpAddress(ipAddress); + + if (entityType == RangerPluginInfo.ENTITY_TYPE_POLICIES) { + pluginSvcVersionInfo.setPolicyActiveVersion(lastKnownVersion); + pluginSvcVersionInfo.setPolicyActivationTime(lastActivationTime); + pluginSvcVersionInfo.setPolicyDownloadedVersion(downloadedVersion); + pluginSvcVersionInfo.setPolicyDownloadTime(new Date().getTime()); + } else { + pluginSvcVersionInfo.setTagActiveVersion(lastKnownVersion); + pluginSvcVersionInfo.setTagActivationTime(lastActivationTime); + pluginSvcVersionInfo.setTagDownloadedVersion(downloadedVersion); + pluginSvcVersionInfo.setTagDownloadTime(new Date().getTime()); + } + + createOrUpdatePluginInfo(pluginSvcVersionInfo, httpCode); + } + + void createOrUpdatePluginInfo(final RangerPluginInfo pluginInfo, final int httpCode) { + if (logger.isDebugEnabled()) { + logger.debug("==> createOrUpdatePluginInfo(pluginInfo=" + pluginInfo + ", httpCode=" + httpCode + ")"); + } + + if (httpCode == HttpServletResponse.SC_NOT_MODIFIED) { + // Create or update PluginInfo record after transaction is completed. If it is created in-line here + // then the TransactionManager will roll-back the changes because the HTTP return code is + // HttpServletResponse.SC_NOT_MODIFIED + Runnable commitWork = new Runnable() { + @Override + public void run() { + doCreateOrUpdateXXPluginInfo(pluginInfo); + } + }; + activityLogger.commitAfterTransactionComplete(commitWork); + } else { + doCreateOrUpdateXXPluginInfo(pluginInfo); + } + if (logger.isDebugEnabled()) { + logger.debug("<== createOrUpdatePluginInfo(pluginInfo=" + pluginInfo + ", httpCode=" + httpCode + ")"); + } + + } + + XXPluginInfo doCreateOrUpdateXXPluginInfo(RangerPluginInfo pluginInfo) { + XXPluginInfo ret = null; + + if (StringUtils.isNotBlank(pluginInfo.getServiceName())) { + + boolean isPolicyInfo = pluginInfo.getPolicyDownloadedVersion() != null; + + // If the ranger-admin is restarted, plugin contains latest version and there is no row for this pluginInfo + if (isPolicyInfo) { + if (pluginInfo.getPolicyDownloadedVersion().equals(pluginInfo.getPolicyActiveVersion())) { + // This is our best guess of when policies may have been downloaded + pluginInfo.setPolicyDownloadTime(pluginInfo.getPolicyActivationTime()); + } + } else { + if (pluginInfo.getTagDownloadedVersion().equals(pluginInfo.getTagActiveVersion())) { + // This is our best guess of when tags may have been downloaded + pluginInfo.setTagDownloadTime(pluginInfo.getTagActivationTime()); + } + } + + XXPluginInfo xObj = rangerDaoManager.getXXPluginInfo().find(pluginInfo.getServiceName(), + pluginInfo.getHostName(), pluginInfo.getAppType()); + + if (xObj == null) { + xObj = pluginInfoService.populateDBObject(pluginInfo); + + if (logger.isDebugEnabled()) { + logger.debug("Creating RangerPluginInfo record for service-version"); + } + ret = rangerDaoManager.getXXPluginInfo().create(xObj); + } else { + boolean needsUpdating = false; + + RangerPluginInfo dbObj = pluginInfoService.populateViewObject(xObj); + if (!dbObj.getIpAddress().equals(pluginInfo.getIpAddress())) { + dbObj.setIpAddress(pluginInfo.getIpAddress()); + needsUpdating = true; + } + if (isPolicyInfo) { + if (dbObj.getPolicyDownloadedVersion() == null || !dbObj.getPolicyDownloadedVersion().equals(pluginInfo.getPolicyDownloadedVersion())) { + dbObj.setPolicyDownloadedVersion(pluginInfo.getPolicyDownloadedVersion()); + dbObj.setPolicyDownloadTime(pluginInfo.getPolicyDownloadTime()); + needsUpdating = true; + } + long lastKnownPolicyVersion = pluginInfo.getPolicyActiveVersion(); + long lastPolicyActivationTime = pluginInfo.getPolicyActivationTime(); + + if (lastKnownPolicyVersion > 0 && (dbObj.getPolicyActiveVersion() == null || !dbObj.getPolicyActiveVersion().equals(lastKnownPolicyVersion))) { + dbObj.setPolicyActiveVersion(lastKnownPolicyVersion); + if (lastPolicyActivationTime > 0) { + dbObj.setPolicyActivationTime(lastPolicyActivationTime); + } + needsUpdating = true; + } else if (lastKnownPolicyVersion == -1) { + dbObj.setPolicyDownloadTime(pluginInfo.getPolicyDownloadTime()); + dbObj.setPolicyActiveVersion(null); + dbObj.setPolicyActivationTime(null); + needsUpdating = true; + } + } else { + if (dbObj.getTagDownloadedVersion() == null || !dbObj.getTagDownloadedVersion().equals(pluginInfo.getTagDownloadedVersion())) { + dbObj.setTagDownloadedVersion(pluginInfo.getTagDownloadedVersion()); + dbObj.setTagDownloadTime(pluginInfo.getTagDownloadTime()); + needsUpdating = true; + } + long lastKnownTagVersion = pluginInfo.getTagActiveVersion(); + long lastTagActivationTime = pluginInfo.getTagActivationTime(); + + if (lastKnownTagVersion > 0 && (dbObj.getTagActiveVersion() == null || !dbObj.getTagActiveVersion().equals(lastKnownTagVersion))) { + dbObj.setTagActiveVersion(lastKnownTagVersion); + if (lastTagActivationTime > 0) { + dbObj.setTagActivationTime(lastTagActivationTime); } - return rangerDaoManager.getXXPolicyExportAudit().create( - xXPolicyExportAudit); + needsUpdating = true; + } else if (lastKnownTagVersion == -1) { + dbObj.setTagDownloadTime(pluginInfo.getTagDownloadTime()); + dbObj.setTagActiveVersion(null); + dbObj.setTagActivationTime(null); + needsUpdating = true; } - }); - return policyExportAudit; + } + + if (needsUpdating) { + if (logger.isDebugEnabled()) { + logger.debug("Updating XXPluginInfo record for service-version"); + } + xObj = pluginInfoService.populateDBObject(dbObj); + + ret = rangerDaoManager.getXXPluginInfo().update(xObj); + } + } + } else { + logger.error("Invalid parameters: pluginInfo=" + pluginInfo + ")"); + } + + return ret; } public VXTrxLogList getReportLogs(SearchCriteria searchCriteria) { @@ -665,17 +842,17 @@ public class AssetMgr extends AssetMgrBase { if (searchCriteria.getParamList() != null && searchCriteria.getParamList().size() > 0) { int clientTimeOffsetInMinute = RestUtil.getClientTimeOffset(); - java.util.Date temp = null; + Date temp = null; DateUtil dateUtil = new DateUtil(); if (searchCriteria.getParamList().containsKey("startDate")) { - temp = (java.util.Date) searchCriteria.getParamList().get( + temp = (Date) searchCriteria.getParamList().get( "startDate"); temp = dateUtil.getDateFromGivenDate(temp, 0, 0, 0, 0); temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); searchCriteria.getParamList().put("startDate", temp); } if (searchCriteria.getParamList().containsKey("endDate")) { - temp = (java.util.Date) searchCriteria.getParamList().get( + temp = (Date) searchCriteria.getParamList().get( "endDate"); temp = dateUtil.getDateFromGivenDate(temp, 0, 23, 59, 59); temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); @@ -713,17 +890,17 @@ public class AssetMgr extends AssetMgrBase { if (searchCriteria.getParamList() != null && searchCriteria.getParamList().size() > 0) { int clientTimeOffsetInMinute = RestUtil.getClientTimeOffset(); - java.util.Date temp = null; + Date temp = null; DateUtil dateUtil = new DateUtil(); if (searchCriteria.getParamList().containsKey("startDate")) { - temp = (java.util.Date) searchCriteria.getParamList().get( + temp = (Date) searchCriteria.getParamList().get( "startDate"); temp = dateUtil.getDateFromGivenDate(temp, 0, 0, 0, 0); temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); searchCriteria.getParamList().put("startDate", temp); } if (searchCriteria.getParamList().containsKey("endDate")) { - temp = (java.util.Date) searchCriteria.getParamList().get( + temp = (Date) searchCriteria.getParamList().get( "endDate"); temp = dateUtil.getDateFromGivenDate(temp, 0, 23, 59, 59); temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); @@ -828,17 +1005,17 @@ public class AssetMgr extends AssetMgrBase { && searchCriteria.getParamList().size() > 0) { int clientTimeOffsetInMinute = RestUtil.getClientTimeOffset(); - java.util.Date temp = null; + Date temp = null; DateUtil dateUtil = new DateUtil(); if (searchCriteria.getParamList().containsKey("startDate")) { - temp = (java.util.Date) searchCriteria.getParamList().get( + temp = (Date) searchCriteria.getParamList().get( "startDate"); temp = dateUtil.getDateFromGivenDate(temp, 0, 0, 0, 0); temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); searchCriteria.getParamList().put("startDate", temp); } if (searchCriteria.getParamList().containsKey("endDate")) { - temp = (java.util.Date) searchCriteria.getParamList().get( + temp = (Date) searchCriteria.getParamList().get( "endDate"); temp = dateUtil.getDateFromGivenDate(temp, 0, 23, 59, 59); temp = dateUtil.addTimeOffset(temp, clientTimeOffsetInMinute); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java index 2d049a4..4fb52a4 100644 --- a/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java +++ b/security-admin/src/main/java/org/apache/ranger/common/RangerSearchUtil.java @@ -67,6 +67,11 @@ public class RangerSearchUtil extends SearchUtil { ret.setParam(SearchFilter.RESOURCE_SIGNATURE, request.getParameter(SearchFilter.RESOURCE_SIGNATURE)); ret.setParam(SearchFilter.POLICY_TYPE, request.getParameter(SearchFilter.POLICY_TYPE)); + ret.setParam(SearchFilter.PLUGIN_HOST_NAME, request.getParameter(SearchFilter.PLUGIN_HOST_NAME)); + ret.setParam(SearchFilter.PLUGIN_APP_TYPE, request.getParameter(SearchFilter.PLUGIN_APP_TYPE)); + ret.setParam(SearchFilter.PLUGIN_ENTITY_TYPE, request.getParameter(SearchFilter.PLUGIN_ENTITY_TYPE)); + ret.setParam(SearchFilter.PLUGIN_IP_ADDRESS, request.getParameter(SearchFilter.PLUGIN_IP_ADDRESS)); + for (Map.Entry e : request.getParameterMap().entrySet()) { String name = e.getKey(); String[] values = e.getValue(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java new file mode 100644 index 0000000..2a62fb4 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/common/db/RangerTransactionSynchronizationAdapter.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.common.db; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionSynchronizationAdapter; +import org.springframework.transaction.support.TransactionSynchronizationManager; +import org.springframework.transaction.support.TransactionTemplate; + +@Component +public class RangerTransactionSynchronizationAdapter extends TransactionSynchronizationAdapter { + + @Autowired + @Qualifier(value = "transactionManager") + PlatformTransactionManager txManager; + + private static final Log LOG = LogFactory.getLog(RangerTransactionSynchronizationAdapter.class); + + private static final ThreadLocal> RUNNABLES = new ThreadLocal>(); + + public void executeOnTransactionCompletion(Runnable runnable) { + if (LOG.isDebugEnabled()) { + LOG.debug("Submitting new runnable {" + runnable + "} to run after completion"); + } + + /* + From TransactionSynchronizationManager documentation: + TransactionSynchronizationManager is a central helper that manages resources and transaction synchronizations per thread. + Resource management code should only register synchronizations when this manager is active, + which can be checked via isSynchronizationActive(); it should perform immediate resource cleanup else. + If transaction synchronization isn't active, there is either no current transaction, + or the transaction manager doesn't support transaction synchronization. + + Note: Synchronization is an Interface for transaction synchronization callbacks which is implemented by + TransactionSynchronizationAdapter + */ + + if (!TransactionSynchronizationManager.isSynchronizationActive()) { + LOG.info("Transaction synchronization is NOT ACTIVE. Executing right now runnable {" + runnable + "}"); + runnable.run(); + return; + } + List threadRunnables = RUNNABLES.get(); + if (threadRunnables == null) { + threadRunnables = new ArrayList(); + RUNNABLES.set(threadRunnables); + // Register a new transaction synchronization for the current thread. + // TransactionSynchronizationManage will call afterCompletion() when current transaction completes. + TransactionSynchronizationManager.registerSynchronization(this); + } + threadRunnables.add(runnable); + } + + @Override + public void afterCompletion(int status) { + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction completed with status {" + (status == STATUS_COMMITTED ? "COMMITTED" : "ROLLED_BACK") + "}"); + } + /* Thread runnables are expected to be executed only when the status is STATUS_ROLLED_BACK. Currently, executeOnTransactionCompletion() + * is called only for those changes that are going to be rolled-back by TransactionSynchronizationManager - such + * as when the operation returns HttpServletResponse.SC_NOT_MODIFIED status. + */ + //if (status == STATUS_ROLLED_BACK) { + final List threadRunnables = RUNNABLES.get(); + if (LOG.isDebugEnabled()) { + LOG.debug("Transaction completed, executing {" + threadRunnables.size() + "} runnables"); + } + if (threadRunnables != null) { + try { + //Create new transaction + TransactionTemplate txTemplate = new TransactionTemplate(txManager); + txTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + + txTemplate.execute(new TransactionCallback() { + public Object doInTransaction(TransactionStatus status) { + for (Runnable runnable : threadRunnables) { + if (LOG.isDebugEnabled()) { + LOG.debug("Executing runnable {" + runnable + "}"); + } + try { + runnable.run(); + } catch (RuntimeException e) { + LOG.error("Failed to execute runnable " + runnable, e); + break; + } + } + + return null; + } + }); + } catch (Exception e) { + LOG.error("Failed to commit TransactionService transaction", e); + LOG.error("Ignoring..."); + } + } + + //} + RUNNABLES.remove(); + } + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java index 6cd5c9f..15331ae 100644 --- a/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java +++ b/security-admin/src/main/java/org/apache/ranger/db/RangerDaoManagerBase.java @@ -363,6 +363,9 @@ public abstract class RangerDaoManagerBase { if (className.equals("XXServiceVersionInfo")) { return getXXServiceVersionInfo(); } + if (className.equals("XXPluginInfo")) { + return getXXPluginInfo(); + } logger.error("No DaoManager found for className=" + className, new Throwable()); return null; } @@ -583,5 +586,9 @@ public abstract class RangerDaoManagerBase { public XXServiceVersionInfoDao getXXServiceVersionInfo() { return new XXServiceVersionInfoDao(this); } + + public XXPluginInfoDao getXXPluginInfo() { + return new XXPluginInfoDao(this); + } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/src/main/java/org/apache/ranger/db/XXPluginInfoDao.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXPluginInfoDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXPluginInfoDao.java new file mode 100644 index 0000000..8eb1636 --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/db/XXPluginInfoDao.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.db; + +import java.util.List; + +import javax.persistence.NoResultException; + +import org.apache.ranger.common.DateUtil; +import org.apache.ranger.common.db.BaseDao; +import org.apache.ranger.entity.XXPluginInfo; + +/** + */ + +public class XXPluginInfoDao extends BaseDao { + /** + * Default Constructor + */ + public XXPluginInfoDao(RangerDaoManagerBase daoManager) { + super(daoManager); + } + + @Override + public XXPluginInfo create(XXPluginInfo obj) { + obj.setCreateTime(DateUtil.getUTCDate()); + obj.setUpdateTime(DateUtil.getUTCDate()); + return super.create(obj); + } + + @Override + public XXPluginInfo update(XXPluginInfo obj) { + obj.setUpdateTime(DateUtil.getUTCDate()); + return super.update(obj); + } + public XXPluginInfo find(String serviceName, String hostName, String appType) { + if (serviceName == null || hostName == null || appType == null) { + return null; + } + try { + return getEntityManager() + .createNamedQuery("XXPluginInfo.find", tClass) + .setParameter("serviceName", serviceName) + .setParameter("appType", appType) + .setParameter("hostName", hostName) + .getSingleResult(); + } catch (NoResultException e) { + return null; + } + } + public List findByServiceName(String serviceName) { + if (serviceName == null) { + return null; + } + try { + return getEntityManager() + .createNamedQuery("XXPluginInfo.findByServiceName", tClass) + .setParameter("serviceName", serviceName).getResultList(); + } catch (NoResultException e) { + return null; + } + } + + public List findByServiceId(Long serviceId) { + if (serviceId == null) { + return null; + } + try { + return getEntityManager() + .createNamedQuery("XXPluginInfo.findByServiceId", tClass) + .setParameter("serviceId", serviceId).getResultList(); + } catch (NoResultException e) { + return null; + } + } + + public List findByServiceAndHostName(String serviceName, String hostName) { + if (serviceName == null || hostName == null) { + return null; + } + try { + return getEntityManager() + .createNamedQuery("XXPluginInfo.findByServiceAndHostName", tClass) + .setParameter("serviceName", serviceName) + .setParameter("hostName", hostName) + .getResultList(); + } catch (NoResultException e) { + return null; + } + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/src/main/java/org/apache/ranger/entity/XXPluginInfo.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/entity/XXPluginInfo.java b/security-admin/src/main/java/org/apache/ranger/entity/XXPluginInfo.java new file mode 100644 index 0000000..2196cdf --- /dev/null +++ b/security-admin/src/main/java/org/apache/ranger/entity/XXPluginInfo.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.entity; + +import java.util.Date; + +import javax.persistence.Cacheable; +import javax.persistence.Entity; +import javax.persistence.Column; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; +import javax.persistence.Temporal; +import javax.persistence.TemporalType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.ranger.common.AppConstants; + +@Entity +@Cacheable +@XmlRootElement +@Table(name = "x_plugin_info") +public class XXPluginInfo implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + @Id + @SequenceGenerator(name = "X_PLUGIN_INFO_SEQ", sequenceName = "X_PLUGIN_INFO_SEQ", allocationSize = 1) + @GeneratedValue(strategy = GenerationType.AUTO, generator = "X_PLUGIN_INFO_SEQ") + @Column(name = "id") + protected Long id; + + @Temporal(TemporalType.TIMESTAMP) + @Column(name="CREATE_TIME" ) + protected Date createTime; + + @Temporal(TemporalType.TIMESTAMP) + @Column(name="UPDATE_TIME" ) + protected Date updateTime; + + @Column(name = "service_name") + protected String serviceName; + + @Column(name = "app_type") + protected String appType; + + @Column(name = "host_name") + protected String hostName; + + @Column(name = "ip_address") + protected String ipAddress; + + @Column(name = "info") + protected String info; + + /** + * Default constructor. This will set all the attributes to default value. + */ + public XXPluginInfo( ) { + } + + public int getMyClassType( ) { + return AppConstants.CLASS_TYPE_NONE; + } + + public String getMyDisplayValue() { + return null; + } + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return this.id; + } + + public void setCreateTime( Date createTime ) { + this.createTime = createTime; + } + + public Date getCreateTime( ) { + return this.createTime; + } + + public void setUpdateTime( Date updateTime ) { + this.updateTime = updateTime; + } + + public Date getUpdateTime( ) { + return this.updateTime; + } + + public void setServiceName(String serviceName) { + this.serviceName = serviceName; + } + + public String getServiceName() { + return this.serviceName; + } + + public void setAppType(String appType) { + this.appType = appType; + } + + public String getAppType() { + return this.appType; + } + + public void setHostName(String hostName) { + this.hostName = hostName; + } + + public String getHostName() { + return this.hostName; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public String getIpAddress() { + return this.ipAddress; + } + + public void setInfo(String info) { + this.info = info; + } + + public String getInfo() { + return this.info; + } + + /** + * This return the bean content in string format + * @return formatedStr + */ + @Override + public String toString( ) { + String str = "XXPluginInfo={"; + str += "id={" + id + "} "; + str += "createTime={" + createTime + "} "; + str += "updateTime={" + updateTime + "} "; + str += "serviceName={" + serviceName + "} "; + str += "hostName={" + hostName + "} "; + str += "appType={" + appType + "} "; + str += "ipAddress={" + ipAddress + "} "; + str += "info={" + info + "} "; + str += "}"; + return str; + } + + /** + * Checks for all attributes except referenced db objects + * @return true if all attributes match + */ + @Override + public boolean equals( Object obj) { + if (obj == null) + return false; + if (this == obj) + return true; + if (!super.equals(obj)) + return false; + if (getClass() != obj.getClass()) + return false; + XXPluginInfo other = (XXPluginInfo) obj; + if ((this.id == null && other.id != null) || (this.id != null && !this.id.equals(other.id))) { + return false; + } + if ((this.createTime == null && other.createTime != null) || (this.createTime != null && !this.createTime.equals(other.createTime))) { + return false; + } + if ((this.updateTime == null && other.updateTime != null) || (this.updateTime != null && !this.updateTime.equals(other.updateTime))) { + return false; + } + if ((this.serviceName == null && other.serviceName != null) || (this.serviceName != null && !this.serviceName.equals(other.serviceName))) { + return false; + } + if ((this.hostName == null && other.hostName != null) || (this.hostName != null && !this.hostName.equals(other.hostName))) { + return false; + } + if ((this.appType == null && other.appType != null) || (this.appType != null && !this.appType.equals(other.appType))) { + return false; + } + if ((this.ipAddress == null && other.ipAddress != null) || (this.ipAddress != null && !this.ipAddress.equals(other.ipAddress))) { + return false; + } + if ((this.info == null && other.info != null) || (this.info != null && !this.info.equals(other.info))) { + return false; + } + return true; + } + + public static boolean equals(Object object1, Object object2) { + if (object1 == object2) { + return true; + } + if ((object1 == null) || (object2 == null)) { + return false; + } + return object1.equals(object2); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java index 3d2997b..52c141b 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/AssetREST.java @@ -535,7 +535,7 @@ public class AssetREST { ServicePolicies servicePolicies = null; try { - servicePolicies = serviceREST.getServicePoliciesIfUpdated(repository, lastKnowPolicyVersion, agentId, request); + servicePolicies = serviceREST.getServicePoliciesIfUpdated(repository, lastKnowPolicyVersion, 0L, agentId, request); } catch(Exception excp) { logger.error("failed to retrieve policies for repository " + repository, excp); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f4a20e0b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java ---------------------------------------------------------------------- diff --git a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java index e2a6435..81d6001 100644 --- a/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java +++ b/security-admin/src/main/java/org/apache/ranger/rest/PublicAPIsv2.java @@ -23,10 +23,12 @@ import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.ranger.common.RESTErrorUtil; import org.apache.ranger.common.annotation.RangerAnnotationJSMgrName; +import org.apache.ranger.plugin.model.RangerPluginInfo; import org.apache.ranger.plugin.model.RangerPolicy; import org.apache.ranger.plugin.model.RangerService; import org.apache.ranger.plugin.model.RangerServiceDef; import org.apache.ranger.plugin.util.SearchFilter; +import org.apache.ranger.view.RangerPluginInfoList; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.security.access.prepost.PreAuthorize; @@ -432,4 +434,19 @@ public class PublicAPIsv2 { logger.debug("<== PublicAPIsv2.deletePolicyByName(" + serviceName + "," + policyName + ")"); } } + + @GET + @Path("/api/plugins/info") + public List getPluginsInfo(@Context HttpServletRequest request) { + if (logger.isDebugEnabled()) { + logger.debug("==> PublicAPIsv2.getPluginsInfo()"); + } + + RangerPluginInfoList pluginInfoList = serviceREST.getPluginsInfo(request); + + if (logger.isDebugEnabled()) { + logger.debug("<== PublicAPIsv2.getPluginsInfo()"); + } + return pluginInfoList.getPluginInfoList(); + } }