Return-Path: X-Original-To: apmail-ranger-commits-archive@www.apache.org Delivered-To: apmail-ranger-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7191D10A66 for ; Tue, 17 Mar 2015 23:33:59 +0000 (UTC) Received: (qmail 45763 invoked by uid 500); 17 Mar 2015 23:33:59 -0000 Delivered-To: apmail-ranger-commits-archive@ranger.apache.org Received: (qmail 45734 invoked by uid 500); 17 Mar 2015 23:33:59 -0000 Mailing-List: contact commits-help@ranger.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@ranger.incubator.apache.org Delivered-To: mailing list commits@ranger.incubator.apache.org Received: (qmail 45702 invoked by uid 99); 17 Mar 2015 23:33:59 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 17 Mar 2015 23:33:59 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED,T_RP_MATCHES_RCVD X-Spam-Check-By: apache.org Received: from [140.211.11.3] (HELO mail.apache.org) (140.211.11.3) by apache.org (qpsmtpd/0.29) with SMTP; Tue, 17 Mar 2015 23:33:44 +0000 Received: (qmail 43372 invoked by uid 99); 17 Mar 2015 23:33:24 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 17 Mar 2015 23:33:24 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id A253BE1862; Tue, 17 Mar 2015 23:33:23 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: bosco@apache.org To: commits@ranger.incubator.apache.org Date: Tue, 17 Mar 2015 23:33:38 -0000 Message-Id: <348eba2cdf354ec9b3cbea0872c7d3dc@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [16/17] incubator-ranger git commit: Support for Solr as Audit Destination. X-Virus-Checked: Checked by ClamAV on apache.org Support for Solr as Audit Destination. Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/40aa090d Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/40aa090d Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/40aa090d Branch: refs/heads/master Commit: 40aa090d7b23a524b9900f5b722d02b48c06b947 Parents: 8e6acd5 Author: Don Bosco Durai Authored: Tue Mar 17 14:17:30 2015 -0700 Committer: Don Bosco Durai Committed: Tue Mar 17 14:17:30 2015 -0700 ---------------------------------------------------------------------- agents-audit/pom.xml | 28 +- .../audit/provider/AuditProviderFactory.java | 23 +- .../audit/provider/BaseAuditProvider.java | 5 +- .../provider/kafka/KafkaAuditProvider.java | 2 +- .../audit/provider/solr/SolrAuditProvider.java | 275 ++++ hbase-agent/conf/ranger-hbase-audit-changes.cfg | 12 + hbase-agent/conf/ranger-hbase-audit.xml | 48 + hbase-agent/scripts/install.properties | 6 + hdfs-agent/conf/ranger-hdfs-audit-changes.cfg | 11 + hdfs-agent/conf/ranger-hdfs-audit.xml | 48 + hdfs-agent/scripts/install.properties | 7 + hive-agent/conf/ranger-hive-audit-changes.cfg | 12 + hive-agent/conf/ranger-hive-audit.xml | 48 + hive-agent/scripts/install.properties | 6 + knox-agent/conf/ranger-knox-audit-changes.cfg | 12 + knox-agent/conf/ranger-knox-audit.xml | 48 + knox-agent/scripts/install.properties | 6 + plugin-yarn/conf/ranger-yarn-audit-changes.cfg | 12 + plugin-yarn/conf/ranger-yarn-audit.xml | 48 + plugin-yarn/scripts/install.properties | 6 + pom.xml | 9 + ranger_solrj/.gitignore | 1 + ranger_solrj/pom.xml | 55 + .../solr/client/solrj/ResponseParser.java | 53 + .../apache/solr/client/solrj/SolrClient.java | 416 ++++++ .../org/apache/solr/client/solrj/SolrQuery.java | 1109 ++++++++++++++++ .../apache/solr/client/solrj/SolrRequest.java | 137 ++ .../apache/solr/client/solrj/SolrResponse.java | 65 + .../apache/solr/client/solrj/SolrServer.java | 25 + .../solr/client/solrj/SolrServerException.java | 54 + .../client/solrj/StreamingResponseCallback.java | 37 + .../client/solrj/beans/BindingException.java | 29 + .../solrj/beans/DocumentObjectBinder.java | 470 +++++++ .../apache/solr/client/solrj/beans/Field.java | 38 + .../solr/client/solrj/beans/package-info.java | 23 + .../client/solrj/impl/BinaryRequestWriter.java | 120 ++ .../client/solrj/impl/BinaryResponseParser.java | 64 + .../solr/client/solrj/impl/CloudSolrClient.java | 1232 ++++++++++++++++++ .../solr/client/solrj/impl/CloudSolrServer.java | 61 + .../solrj/impl/ConcurrentUpdateSolrClient.java | 492 +++++++ .../solrj/impl/ConcurrentUpdateSolrServer.java | 46 + .../client/solrj/impl/HttpClientConfigurer.java | 97 ++ .../solr/client/solrj/impl/HttpClientUtil.java | 365 ++++++ .../solr/client/solrj/impl/HttpSolrClient.java | 821 ++++++++++++ .../solr/client/solrj/impl/HttpSolrServer.java | 41 + .../client/solrj/impl/LBHttpSolrClient.java | 730 +++++++++++ .../client/solrj/impl/LBHttpSolrServer.java | 43 + .../client/solrj/impl/NoOpResponseParser.java | 83 ++ .../impl/StreamingBinaryResponseParser.java | 91 ++ .../client/solrj/impl/XMLResponseParser.java | 465 +++++++ .../solr/client/solrj/impl/package-info.java | 24 + .../apache/solr/client/solrj/package-info.java | 23 + .../solrj/request/AbstractUpdateRequest.java | 144 ++ .../solrj/request/CollectionAdminRequest.java | 860 ++++++++++++ .../request/ContentStreamUpdateRequest.java | 78 ++ .../client/solrj/request/CoreAdminRequest.java | 593 +++++++++ .../client/solrj/request/DirectXmlRequest.java | 66 + .../solrj/request/DocumentAnalysisRequest.java | 199 +++ .../solrj/request/FieldAnalysisRequest.java | 270 ++++ .../client/solrj/request/IsUpdateRequest.java | 26 + .../request/JavaBinUpdateRequestCodec.java | 251 ++++ .../solr/client/solrj/request/LukeRequest.java | 120 ++ .../solr/client/solrj/request/QueryRequest.java | 89 ++ .../client/solrj/request/RequestWriter.java | 146 +++ .../solr/client/solrj/request/SolrPing.java | 111 ++ .../client/solrj/request/UpdateRequest.java | 463 +++++++ .../solr/client/solrj/request/package-info.java | 23 + .../solrj/response/AnalysisResponseBase.java | 252 ++++ .../solrj/response/CollectionAdminResponse.java | 79 ++ .../solrj/response/CoreAdminResponse.java | 58 + .../response/DocumentAnalysisResponse.java | 258 ++++ .../solr/client/solrj/response/FacetField.java | 176 +++ .../solrj/response/FieldAnalysisResponse.java | 204 +++ .../client/solrj/response/FieldStatsInfo.java | 191 +++ .../solr/client/solrj/response/Group.java | 69 + .../client/solrj/response/GroupCommand.java | 125 ++ .../client/solrj/response/GroupResponse.java | 56 + .../client/solrj/response/IntervalFacet.java | 85 ++ .../client/solrj/response/LukeResponse.java | 270 ++++ .../solr/client/solrj/response/PivotField.java | 97 ++ .../client/solrj/response/QueryResponse.java | 586 +++++++++ .../solr/client/solrj/response/RangeFacet.java | 126 ++ .../client/solrj/response/SolrPingResponse.java | 28 + .../client/solrj/response/SolrResponseBase.java | 91 ++ .../solrj/response/SpellCheckResponse.java | 273 ++++ .../client/solrj/response/TermsResponse.java | 89 ++ .../client/solrj/response/UpdateResponse.java | 30 + .../client/solrj/response/package-info.java | 24 + .../solr/client/solrj/util/ClientUtils.java | 251 ++++ .../solr/client/solrj/util/package-info.java | 23 + .../org/apache/solr/common/EnumFieldValue.java | 116 ++ .../org/apache/solr/common/SolrDocument.java | 396 ++++++ .../apache/solr/common/SolrDocumentList.java | 68 + .../org/apache/solr/common/SolrException.java | 208 +++ .../apache/solr/common/SolrInputDocument.java | 301 +++++ .../org/apache/solr/common/SolrInputField.java | 232 ++++ .../org/apache/solr/common/StringUtils.java | 26 + .../org/apache/solr/common/cloud/Aliases.java | 63 + .../solr/common/cloud/BeforeReconnect.java | 22 + .../solr/common/cloud/ClosableThread.java | 27 + .../apache/solr/common/cloud/ClusterState.java | 397 ++++++ .../solr/common/cloud/ClusterStateUtil.java | 230 ++++ .../solr/common/cloud/CompositeIdRouter.java | 327 +++++ .../solr/common/cloud/ConnectionManager.java | 237 ++++ .../common/cloud/DefaultConnectionStrategy.java | 75 ++ .../solr/common/cloud/DefaultZkACLProvider.java | 45 + .../cloud/DefaultZkCredentialsProvider.java | 41 + .../apache/solr/common/cloud/DocCollection.java | 201 +++ .../org/apache/solr/common/cloud/DocRouter.java | 227 ++++ .../solr/common/cloud/HashBasedRouter.java | 81 ++ .../solr/common/cloud/ImplicitDocRouter.java | 104 ++ .../apache/solr/common/cloud/OnReconnect.java | 22 + .../apache/solr/common/cloud/PlainIdRouter.java | 23 + .../org/apache/solr/common/cloud/Replica.java | 48 + .../apache/solr/common/cloud/RoutingRule.java | 71 + .../solr/common/cloud/SaslZkACLProvider.java | 49 + .../org/apache/solr/common/cloud/Slice.java | 196 +++ .../apache/solr/common/cloud/SolrZkClient.java | 736 +++++++++++ .../apache/solr/common/cloud/SolrZooKeeper.java | 103 ++ ...ParamsAllAndReadonlyDigestZkACLProvider.java | 89 ++ ...tCredentialsDigestZkCredentialsProvider.java | 60 + .../apache/solr/common/cloud/ZkACLProvider.java | 28 + .../cloud/ZkClientConnectionStrategy.java | 113 ++ .../apache/solr/common/cloud/ZkCmdExecutor.java | 111 ++ .../solr/common/cloud/ZkConfigManager.java | 145 +++ .../solr/common/cloud/ZkCoreNodeProps.java | 74 ++ .../common/cloud/ZkCredentialsProvider.java | 45 + .../apache/solr/common/cloud/ZkNodeProps.java | 154 +++ .../apache/solr/common/cloud/ZkOperation.java | 37 + .../apache/solr/common/cloud/ZkStateReader.java | 925 +++++++++++++ .../solr/common/cloud/ZooKeeperException.java | 33 + .../apache/solr/common/cloud/package-info.java | 23 + .../org/apache/solr/common/luke/FieldFlag.java | 70 + .../apache/solr/common/luke/package-info.java | 23 + .../org/apache/solr/common/package-info.java | 23 + .../solr/common/params/AnalysisParams.java | 60 + .../solr/common/params/AppendedSolrParams.java | 55 + .../solr/common/params/CollectionParams.java | 74 ++ .../apache/solr/common/params/CommonParams.java | 228 ++++ .../solr/common/params/CoreAdminParams.java | 151 +++ .../solr/common/params/CursorMarkParams.java | 48 + .../solr/common/params/DefaultSolrParams.java | 68 + .../apache/solr/common/params/DisMaxParams.java | 78 ++ .../apache/solr/common/params/EventParams.java | 29 + .../apache/solr/common/params/ExpandParams.java | 32 + .../apache/solr/common/params/FacetParams.java | 405 ++++++ .../apache/solr/common/params/GroupParams.java | 71 + .../solr/common/params/HighlightParams.java | 82 ++ .../solr/common/params/MapSolrParams.java | 88 ++ .../common/params/ModifiableSolrParams.java | 210 +++ .../solr/common/params/MoreLikeThisParams.java | 74 ++ .../solr/common/params/MultiMapSolrParams.java | 92 ++ .../common/params/QueryElevationParams.java | 53 + .../solr/common/params/RequiredSolrParams.java | 155 +++ .../apache/solr/common/params/ShardParams.java | 56 + .../apache/solr/common/params/SimpleParams.java | 50 + .../apache/solr/common/params/SolrParams.java | 363 ++++++ .../solr/common/params/SpatialParams.java | 41 + .../solr/common/params/SpellingParams.java | 174 +++ .../apache/solr/common/params/StatsParams.java | 28 + .../solr/common/params/TermVectorParams.java | 66 + .../apache/solr/common/params/TermsParams.java | 120 ++ .../apache/solr/common/params/UpdateParams.java | 72 + .../apache/solr/common/params/package-info.java | 22 + .../org/apache/solr/common/util/Base64.java | 153 +++ .../org/apache/solr/common/util/ByteUtils.java | 126 ++ .../apache/solr/common/util/ContentStream.java | 81 ++ .../solr/common/util/ContentStreamBase.java | 260 ++++ .../solr/common/util/DataInputInputStream.java | 27 + .../org/apache/solr/common/util/DateUtil.java | 260 ++++ .../apache/solr/common/util/ExecutorUtil.java | 64 + .../solr/common/util/FastInputStream.java | 253 ++++ .../solr/common/util/FastOutputStream.java | 233 ++++ .../java/org/apache/solr/common/util/Hash.java | 422 ++++++ .../org/apache/solr/common/util/IOUtils.java | 37 + .../apache/solr/common/util/IteratorChain.java | 87 ++ .../apache/solr/common/util/JavaBinCodec.java | 820 ++++++++++++ .../solr/common/util/JsonRecordReader.java | 586 +++++++++ .../org/apache/solr/common/util/NamedList.java | 708 ++++++++++ .../solr/common/util/ObjectReleaseTracker.java | 62 + .../org/apache/solr/common/util/RetryUtil.java | 43 + .../solr/common/util/SimpleOrderedMap.java | 67 + .../common/util/SolrjNamedThreadFactory.java | 50 + .../org/apache/solr/common/util/StrUtils.java | 309 +++++ .../org/apache/solr/common/util/URLUtil.java | 50 + .../java/org/apache/solr/common/util/XML.java | 207 +++ .../apache/solr/common/util/XMLErrorLogger.java | 84 ++ .../apache/solr/common/util/package-info.java | 23 + ranger_solrj/src/main/java/overview.html | 21 + security-admin/pom.xml | 5 + security-admin/scripts/install.properties | 10 + security-admin/scripts/setup.sh | 48 +- .../java/org/apache/ranger/biz/AssetMgr.java | 11 +- .../org/apache/ranger/biz/RangerBizUtil.java | 434 +++--- .../java/org/apache/ranger/biz/XAuditMgr.java | 41 +- .../apache/ranger/db/RangerDaoManagerBase.java | 3 + .../ranger/solr/SolrAccessAuditsService.java | 253 ++++ .../java/org/apache/ranger/solr/SolrMgr.java | 99 ++ .../java/org/apache/ranger/solr/SolrUtil.java | 327 +++++ src/main/assembly/hbase-agent.xml | 6 + src/main/assembly/hdfs-agent.xml | 6 + src/main/assembly/hive-agent.xml | 6 + src/main/assembly/knox-agent.xml | 6 + src/main/assembly/storm-agent.xml | 6 + storm-agent/conf/ranger-storm-audit-changes.cfg | 13 + storm-agent/conf/ranger-storm-audit.xml | 48 + storm-agent/scripts/install.properties | 6 + 207 files changed, 30674 insertions(+), 219 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/pom.xml ---------------------------------------------------------------------- diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml index e54ec36..6715575 100644 --- a/agents-audit/pom.xml +++ b/agents-audit/pom.xml @@ -63,9 +63,29 @@ ${log4j.version} - org.apache.kafka - kafka_2.10 - 0.8.2.0 - + org.apache.kafka + kafka_2.10 + ${kafka.version} + + + org.apache.ranger + ranger_solrj + ${ranger.solrj.version} + + + org.apache.httpcomponents + httpclient + ${httpcomponent.httpclient.version} + + + org.apache.httpcomponents + httpmime + ${httpcomponent.httpmime.version} + + + org.noggit + noggit + ${noggit.version} + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java index 9fbe29f..bb8fa6d 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java @@ -26,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider; import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider; +import org.apache.ranger.audit.provider.solr.SolrAuditProvider; /* @@ -43,6 +44,7 @@ public class AuditProviderFactory { private static final String AUDIT_HDFS_IS_ENABLED_PROP = "xasecure.audit.hdfs.is.enabled"; private static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled" ; private static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled"; + private static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled"; private static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT = 10 * 1024; private static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT = 5 * 1000; @@ -99,8 +101,9 @@ public class AuditProviderFactory { boolean isAuditToHdfsEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false); boolean isAuditToLog4jEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false); boolean isAuditToKafkaEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_KAFKA_IS_ENABLED_PROP, false); + boolean isAuditToSolrEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_SOLR_IS_ENABLED_PROP, false); - if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled)) { + if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled || isAuditToSolrEnabled)) { LOG.info("AuditProviderFactory: Audit not enabled.."); mProvider = getDefaultProvider(); @@ -111,6 +114,7 @@ public class AuditProviderFactory { List providers = new ArrayList(); if(isAuditToDbEnabled) { + LOG.info("DbAuditProvider is enabled"); DbAuditProvider dbProvider = new DbAuditProvider(); boolean isAuditToDbAsync = BaseAuditProvider.getBooleanProperty(props, DbAuditProvider.AUDIT_DB_IS_ASYNC_PROP, false); @@ -128,6 +132,8 @@ public class AuditProviderFactory { } if(isAuditToHdfsEnabled) { + LOG.info("HdfsAuditProvider is enabled"); + HdfsAuditProvider hdfsProvider = new HdfsAuditProvider(); boolean isAuditToHdfsAsync = BaseAuditProvider.getBooleanProperty(props, HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false); @@ -156,7 +162,20 @@ public class AuditProviderFactory { providers.add(kafkaProvider); } } - + + if(isAuditToSolrEnabled) { + LOG.info("SolrAuditProvider is enabled"); + SolrAuditProvider solrProvider = new SolrAuditProvider(); + solrProvider.init(props); + + if( solrProvider.isAsync()) { + AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MySolrAuditProvider", solrProvider.getMaxQueueSize(), solrProvider.getMaxFlushInterval(), solrProvider); + providers.add(asyncProvider); + } else { + providers.add(solrProvider); + } + } + if(isAuditToLog4jEnabled) { Log4jAuditProvider log4jProvider = new Log4jAuditProvider(); http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java index 14e6220..a068b8f 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java @@ -40,7 +40,7 @@ public abstract class BaseAuditProvider implements AuditProvider { private int maxQueueSize = AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT; private int maxFlushInterval = AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT; - + protected Properties props = null; public BaseAuditProvider() { } @@ -48,7 +48,8 @@ public abstract class BaseAuditProvider implements AuditProvider { @Override public void init(Properties props) { LOG.info("BaseAuditProvider.init()"); - + this.props = props; + mLogFailureReportMinIntervalInMs = getIntProperty(props, AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000); } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java index 54e73ea..0ec8790 100644 --- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java @@ -117,7 +117,7 @@ public class KafkaAuditProvider extends BaseAuditProvider { } } catch (Throwable t) { LOG.error("Error sending message to Kafka topic. topic=" + topic - + ", message=" + message); + + ", message=" + message, t); } } http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java ---------------------------------------------------------------------- diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java new file mode 100644 index 0000000..1b463e6 --- /dev/null +++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java @@ -0,0 +1,275 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.audit.provider.solr; + +import java.util.Date; +import java.util.Properties; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.audit.model.AuditEventBase; +import org.apache.ranger.audit.model.AuthzAuditEvent; +import org.apache.ranger.audit.provider.BaseAuditProvider; +import org.apache.ranger.audit.provider.MiscUtil; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.common.SolrInputDocument; + +public class SolrAuditProvider extends BaseAuditProvider { + private static final Log LOG = LogFactory.getLog(SolrAuditProvider.class); + + public static final String AUDIT_MAX_QUEUE_SIZE_PROP = "xasecure.audit.solr.async.max.queue.size"; + public static final String AUDIT_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.solr.async.max.flush.interval.ms"; + public static final String AUDIT_RETRY_WAIT_PROP = "xasecure.audit.solr.retry.ms"; + + static final Object lock = new Object(); + SolrClient solrClient = null; + Date lastConnectTime = null; + long lastFailTime = 0; + + int retryWaitTime = 30000; + + public SolrAuditProvider() { + } + + @Override + public void init(Properties props) { + LOG.info("init() called"); + super.init(props); + + setMaxQueueSize(BaseAuditProvider.getIntProperty(props, + AUDIT_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT)); + setMaxFlushInterval(BaseAuditProvider.getIntProperty(props, + AUDIT_MAX_QUEUE_SIZE_PROP, + AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT)); + retryWaitTime = BaseAuditProvider.getIntProperty(props, + AUDIT_RETRY_WAIT_PROP, retryWaitTime); + } + + void connect() { + if (solrClient == null) { + synchronized (lock) { + if (solrClient == null) { + String solrURL = BaseAuditProvider.getStringProperty(props, + "xasecure.audit.solr.solr_url"); + + if (lastConnectTime != null) { + // Let's wait for enough time before retrying + long diff = System.currentTimeMillis() + - lastConnectTime.getTime(); + if (diff < retryWaitTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignore connecting to solr url=" + + solrURL + ", lastConnect=" + diff + + "ms"); + } + return; + } + } + lastConnectTime = new Date(); + + if (solrURL == null || solrURL.isEmpty()) { + LOG.fatal("Solr URL for Audit is empty"); + return; + } + + try { + // TODO: Need to support SolrCloud also + solrClient = new HttpSolrClient(solrURL); + if (solrClient instanceof HttpSolrClient) { + HttpSolrClient httpSolrClient = (HttpSolrClient) solrClient; + httpSolrClient.setAllowCompression(true); + httpSolrClient.setConnectionTimeout(1000); + // solrClient.setSoTimeout(10000); + httpSolrClient.setMaxRetries(1); + } + } catch (Throwable t) { + LOG.fatal("Can't connect to Solr server. URL=" + + solrURL, t); + } + } + } + } + } + + /* + * (non-Javadoc) + * + * @see + * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger. + * audit.model.AuditEventBase) + */ + @Override + public void log(AuditEventBase event) { + if (!(event instanceof AuthzAuditEvent)) { + LOG.error(event.getClass().getName() + + " audit event class type is not supported"); + return; + } + AuthzAuditEvent authzEvent = (AuthzAuditEvent) event; + // TODO: This should be done at a higher level + + if (authzEvent.getAgentHostname() == null) { + authzEvent.setAgentHostname(MiscUtil.getHostname()); + } + + if (authzEvent.getLogType() == null) { + authzEvent.setLogType("RangerAudit"); + } + + if (authzEvent.getEventId() == null) { + authzEvent.setEventId(MiscUtil.generateUniqueId()); + } + + try { + if (solrClient == null) { + connect(); + if (solrClient == null) { + // Solr is still not initialized. So need to throw error + return; + } + } + + if (lastFailTime > 0) { + long diff = System.currentTimeMillis() - lastFailTime; + if (diff < retryWaitTime) { + if (LOG.isDebugEnabled()) { + LOG.debug("Ignore sending audit. lastConnect=" + diff + + " ms"); + } + return; + } + } + // Convert AuditEventBase to Solr document + SolrInputDocument document = toSolrDoc(authzEvent); + UpdateResponse response = solrClient.add(document); + if (response.getStatus() != 0) { + lastFailTime = System.currentTimeMillis(); + + // System.out.println("Response=" + response.toString() + // + ", status= " + response.getStatus() + ", event=" + // + event); + // throw new Exception("Aborting. event=" + event + + // ", response=" + // + response.toString()); + } else { + lastFailTime = 0; + } + + } catch (Throwable t) { + LOG.error("Error sending message to Solr", t); + } + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#start() + */ + @Override + public void start() { + connect(); + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#stop() + */ + @Override + public void stop() { + // TODO Auto-generated method stub + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete() + */ + @Override + public void waitToComplete() { + // TODO Auto-generated method stub + + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#isFlushPending() + */ + @Override + public boolean isFlushPending() { + // TODO Auto-generated method stub + return false; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#getLastFlushTime() + */ + @Override + public long getLastFlushTime() { + // TODO Auto-generated method stub + return 0; + } + + /* + * (non-Javadoc) + * + * @see org.apache.ranger.audit.provider.AuditProvider#flush() + */ + @Override + public void flush() { + // TODO Auto-generated method stub + + } + + SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", auditEvent.getEventId()); + doc.addField("access", auditEvent.getAccessType()); + doc.addField("enforcer", auditEvent.getAclEnforcer()); + doc.addField("agent", auditEvent.getAgentId()); + doc.addField("repo", auditEvent.getRepositoryName()); + doc.addField("sess", auditEvent.getSessionId()); + doc.addField("reqUser", auditEvent.getUser()); + doc.addField("reqData", auditEvent.getRequestData()); + doc.addField("resource", auditEvent.getResourcePath()); + doc.addField("cliIP", auditEvent.getClientIP()); + doc.addField("logType", auditEvent.getLogType()); + doc.addField("result", auditEvent.getAccessResult()); + doc.addField("policy", auditEvent.getPolicyId()); + doc.addField("repoType", auditEvent.getRepositoryType()); + doc.addField("resType", auditEvent.getResourceType()); + doc.addField("reason", auditEvent.getResultReason()); + doc.addField("action", auditEvent.getAction()); + doc.addField("evtTime", auditEvent.getEventTime()); + return doc; + } + + public boolean isAsync() { + return true; + } + +} http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg index e5c381a..221d20a 100644 --- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg +++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg @@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit.xml ---------------------------------------------------------------------- diff --git a/hbase-agent/conf/ranger-hbase-audit.xml b/hbase-agent/conf/ranger-hbase-audit.xml index b39696b..e5bfb89 100644 --- a/hbase-agent/conf/ranger-hbase-audit.xml +++ b/hbase-agent/conf/ranger-hbase-audit.xml @@ -183,4 +183,52 @@ xasecure.audit.log4j.async.max.flush.interval.ms 30000 + + + + xasecure.audit.kafka.is.enabled + false + + + + xasecure.audit.kafka.async.max.queue.size + 1 + + + + xasecure.audit.kafka.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.kafka.broker_list + localhost:9092 + + + + xasecure.audit.kafka.topic_name + ranger_audits + + + + + xasecure.audit.ranger.is.enabled + false + + + + xasecure.audit.ranger.async.max.queue.size + 1 + + + + xasecure.audit.ranger.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.solr.solr_url + http://localhost:6083/solr/ranger_audits + + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties index 5a81ad4..7ff29c9 100644 --- a/hbase-agent/scripts/install.properties +++ b/hbase-agent/scripts/install.properties @@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg index e5c381a..8d31016 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg +++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg @@ -31,3 +31,14 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit.xml ---------------------------------------------------------------------- diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml index d26345d..09114ad 100644 --- a/hdfs-agent/conf/ranger-hdfs-audit.xml +++ b/hdfs-agent/conf/ranger-hdfs-audit.xml @@ -183,4 +183,52 @@ xasecure.audit.log4j.async.max.flush.interval.ms 30000 + + + + xasecure.audit.kafka.is.enabled + false + + + + xasecure.audit.kafka.async.max.queue.size + 1 + + + + xasecure.audit.kafka.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.kafka.broker_list + localhost:9092 + + + + xasecure.audit.kafka.topic_name + ranger_audits + + + + + xasecure.audit.ranger.is.enabled + false + + + + xasecure.audit.ranger.async.max.queue.size + 1 + + + + xasecure.audit.ranger.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.solr.solr_url + http://localhost:6083/solr/ranger_audits + + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties index 93790e3..2e1b61a 100644 --- a/hdfs-agent/scripts/install.properties +++ b/hdfs-agent/scripts/install.properties @@ -89,6 +89,13 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg index 9fa7608..2d6d414 100644 --- a/hive-agent/conf/ranger-hive-audit-changes.cfg +++ b/hive-agent/conf/ranger-hive-audit-changes.cfg @@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit.xml ---------------------------------------------------------------------- diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml index d011b24..e753336 100644 --- a/hive-agent/conf/ranger-hive-audit.xml +++ b/hive-agent/conf/ranger-hive-audit.xml @@ -183,4 +183,52 @@ xasecure.audit.log4j.async.max.flush.interval.ms 30000 + + + + xasecure.audit.kafka.is.enabled + false + + + + xasecure.audit.kafka.async.max.queue.size + 1 + + + + xasecure.audit.kafka.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.kafka.broker_list + localhost:9092 + + + + xasecure.audit.kafka.topic_name + ranger_audits + + + + + xasecure.audit.ranger.is.enabled + false + + + + xasecure.audit.ranger.async.max.queue.size + 1 + + + + xasecure.audit.ranger.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.solr.solr_url + http://localhost:6083/solr/ranger_audits + + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties index bbd1849..75b1b5d 100644 --- a/hive-agent/scripts/install.properties +++ b/hive-agent/scripts/install.properties @@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg index 7ae334e..f97d10f 100644 --- a/knox-agent/conf/ranger-knox-audit-changes.cfg +++ b/knox-agent/conf/ranger-knox-audit-changes.cfg @@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit.xml ---------------------------------------------------------------------- diff --git a/knox-agent/conf/ranger-knox-audit.xml b/knox-agent/conf/ranger-knox-audit.xml index 0fdcefc..6f0adb9 100644 --- a/knox-agent/conf/ranger-knox-audit.xml +++ b/knox-agent/conf/ranger-knox-audit.xml @@ -183,4 +183,52 @@ xasecure.audit.log4j.async.max.flush.interval.ms 30000 + + + + xasecure.audit.kafka.is.enabled + false + + + + xasecure.audit.kafka.async.max.queue.size + 1 + + + + xasecure.audit.kafka.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.kafka.broker_list + localhost:9092 + + + + xasecure.audit.kafka.topic_name + ranger_audits + + + + + xasecure.audit.ranger.is.enabled + false + + + + xasecure.audit.ranger.async.max.queue.size + 1 + + + + xasecure.audit.ranger.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.solr.solr_url + http://localhost:6083/solr/ranger_audits + + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/scripts/install.properties ---------------------------------------------------------------------- diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties index d821c5d..ecd9813 100644 --- a/knox-agent/scripts/install.properties +++ b/knox-agent/scripts/install.properties @@ -92,6 +92,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg index 4f2c5a2..e0dbea2 100644 --- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg +++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg @@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit.xml ---------------------------------------------------------------------- diff --git a/plugin-yarn/conf/ranger-yarn-audit.xml b/plugin-yarn/conf/ranger-yarn-audit.xml index c0096a4..f1e9687 100644 --- a/plugin-yarn/conf/ranger-yarn-audit.xml +++ b/plugin-yarn/conf/ranger-yarn-audit.xml @@ -184,4 +184,52 @@ xasecure.audit.log4j.async.max.flush.interval.ms 30000 + + + + xasecure.audit.kafka.is.enabled + false + + + + xasecure.audit.kafka.async.max.queue.size + 1 + + + + xasecure.audit.kafka.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.kafka.broker_list + localhost:9092 + + + + xasecure.audit.kafka.topic_name + ranger_audits + + + + + xasecure.audit.ranger.is.enabled + false + + + + xasecure.audit.ranger.async.max.queue.size + 1 + + + + xasecure.audit.ranger.async.max.flush.interval.ms + 1000 + + + + xasecure.audit.solr.solr_url + http://localhost:6083/solr/ranger_audits + + http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/scripts/install.properties ---------------------------------------------------------------------- diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties index d2d1ffe..bbe9f7f 100644 --- a/plugin-yarn/scripts/install.properties +++ b/plugin-yarn/scripts/install.properties @@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + # # SSL Client Certificate Information # http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index ef39d37..00c8339 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,7 @@ knox-agent storm-agent plugin-yarn + ranger_solrj security-admin ugsync unixauthclient @@ -129,6 +130,9 @@ 1.3 0.99.2 1.2.0-SNAPSHOT + 4.2.5 + 4.2.5 + 4.2.5 0.9.2-incubating 0.5.2 3.12.1.GA @@ -138,16 +142,20 @@ 1.17.1 2.6 4.11 + 0.8.2.0 1.8.4 1.3 0.5.0 ${project.basedir}/../lib/local 1.2.17 5.1.31 + 0.6 r239 UTF-8 hadoop-security/plugins 1.7.5 + + 0.4.0 2.5.6