ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [16/17] incubator-ranger git commit: Support for Solr as Audit Destination.
Date Tue, 17 Mar 2015 23:33:38 GMT
Support for Solr as Audit Destination.

Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/40aa090d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/40aa090d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/40aa090d

Branch: refs/heads/master
Commit: 40aa090d7b23a524b9900f5b722d02b48c06b947
Parents: 8e6acd5
Author: Don Bosco Durai <bosco@apache.org>
Authored: Tue Mar 17 14:17:30 2015 -0700
Committer: Don Bosco Durai <bosco@apache.org>
Committed: Tue Mar 17 14:17:30 2015 -0700

----------------------------------------------------------------------
 agents-audit/pom.xml                            |   28 +-
 .../audit/provider/AuditProviderFactory.java    |   23 +-
 .../audit/provider/BaseAuditProvider.java       |    5 +-
 .../provider/kafka/KafkaAuditProvider.java      |    2 +-
 .../audit/provider/solr/SolrAuditProvider.java  |  275 ++++
 hbase-agent/conf/ranger-hbase-audit-changes.cfg |   12 +
 hbase-agent/conf/ranger-hbase-audit.xml         |   48 +
 hbase-agent/scripts/install.properties          |    6 +
 hdfs-agent/conf/ranger-hdfs-audit-changes.cfg   |   11 +
 hdfs-agent/conf/ranger-hdfs-audit.xml           |   48 +
 hdfs-agent/scripts/install.properties           |    7 +
 hive-agent/conf/ranger-hive-audit-changes.cfg   |   12 +
 hive-agent/conf/ranger-hive-audit.xml           |   48 +
 hive-agent/scripts/install.properties           |    6 +
 knox-agent/conf/ranger-knox-audit-changes.cfg   |   12 +
 knox-agent/conf/ranger-knox-audit.xml           |   48 +
 knox-agent/scripts/install.properties           |    6 +
 plugin-yarn/conf/ranger-yarn-audit-changes.cfg  |   12 +
 plugin-yarn/conf/ranger-yarn-audit.xml          |   48 +
 plugin-yarn/scripts/install.properties          |    6 +
 pom.xml                                         |    9 +
 ranger_solrj/.gitignore                         |    1 +
 ranger_solrj/pom.xml                            |   55 +
 .../solr/client/solrj/ResponseParser.java       |   53 +
 .../apache/solr/client/solrj/SolrClient.java    |  416 ++++++
 .../org/apache/solr/client/solrj/SolrQuery.java | 1109 ++++++++++++++++
 .../apache/solr/client/solrj/SolrRequest.java   |  137 ++
 .../apache/solr/client/solrj/SolrResponse.java  |   65 +
 .../apache/solr/client/solrj/SolrServer.java    |   25 +
 .../solr/client/solrj/SolrServerException.java  |   54 +
 .../client/solrj/StreamingResponseCallback.java |   37 +
 .../client/solrj/beans/BindingException.java    |   29 +
 .../solrj/beans/DocumentObjectBinder.java       |  470 +++++++
 .../apache/solr/client/solrj/beans/Field.java   |   38 +
 .../solr/client/solrj/beans/package-info.java   |   23 +
 .../client/solrj/impl/BinaryRequestWriter.java  |  120 ++
 .../client/solrj/impl/BinaryResponseParser.java |   64 +
 .../solr/client/solrj/impl/CloudSolrClient.java | 1232 ++++++++++++++++++
 .../solr/client/solrj/impl/CloudSolrServer.java |   61 +
 .../solrj/impl/ConcurrentUpdateSolrClient.java  |  492 +++++++
 .../solrj/impl/ConcurrentUpdateSolrServer.java  |   46 +
 .../client/solrj/impl/HttpClientConfigurer.java |   97 ++
 .../solr/client/solrj/impl/HttpClientUtil.java  |  365 ++++++
 .../solr/client/solrj/impl/HttpSolrClient.java  |  821 ++++++++++++
 .../solr/client/solrj/impl/HttpSolrServer.java  |   41 +
 .../client/solrj/impl/LBHttpSolrClient.java     |  730 +++++++++++
 .../client/solrj/impl/LBHttpSolrServer.java     |   43 +
 .../client/solrj/impl/NoOpResponseParser.java   |   83 ++
 .../impl/StreamingBinaryResponseParser.java     |   91 ++
 .../client/solrj/impl/XMLResponseParser.java    |  465 +++++++
 .../solr/client/solrj/impl/package-info.java    |   24 +
 .../apache/solr/client/solrj/package-info.java  |   23 +
 .../solrj/request/AbstractUpdateRequest.java    |  144 ++
 .../solrj/request/CollectionAdminRequest.java   |  860 ++++++++++++
 .../request/ContentStreamUpdateRequest.java     |   78 ++
 .../client/solrj/request/CoreAdminRequest.java  |  593 +++++++++
 .../client/solrj/request/DirectXmlRequest.java  |   66 +
 .../solrj/request/DocumentAnalysisRequest.java  |  199 +++
 .../solrj/request/FieldAnalysisRequest.java     |  270 ++++
 .../client/solrj/request/IsUpdateRequest.java   |   26 +
 .../request/JavaBinUpdateRequestCodec.java      |  251 ++++
 .../solr/client/solrj/request/LukeRequest.java  |  120 ++
 .../solr/client/solrj/request/QueryRequest.java |   89 ++
 .../client/solrj/request/RequestWriter.java     |  146 +++
 .../solr/client/solrj/request/SolrPing.java     |  111 ++
 .../client/solrj/request/UpdateRequest.java     |  463 +++++++
 .../solr/client/solrj/request/package-info.java |   23 +
 .../solrj/response/AnalysisResponseBase.java    |  252 ++++
 .../solrj/response/CollectionAdminResponse.java |   79 ++
 .../solrj/response/CoreAdminResponse.java       |   58 +
 .../response/DocumentAnalysisResponse.java      |  258 ++++
 .../solr/client/solrj/response/FacetField.java  |  176 +++
 .../solrj/response/FieldAnalysisResponse.java   |  204 +++
 .../client/solrj/response/FieldStatsInfo.java   |  191 +++
 .../solr/client/solrj/response/Group.java       |   69 +
 .../client/solrj/response/GroupCommand.java     |  125 ++
 .../client/solrj/response/GroupResponse.java    |   56 +
 .../client/solrj/response/IntervalFacet.java    |   85 ++
 .../client/solrj/response/LukeResponse.java     |  270 ++++
 .../solr/client/solrj/response/PivotField.java  |   97 ++
 .../client/solrj/response/QueryResponse.java    |  586 +++++++++
 .../solr/client/solrj/response/RangeFacet.java  |  126 ++
 .../client/solrj/response/SolrPingResponse.java |   28 +
 .../client/solrj/response/SolrResponseBase.java |   91 ++
 .../solrj/response/SpellCheckResponse.java      |  273 ++++
 .../client/solrj/response/TermsResponse.java    |   89 ++
 .../client/solrj/response/UpdateResponse.java   |   30 +
 .../client/solrj/response/package-info.java     |   24 +
 .../solr/client/solrj/util/ClientUtils.java     |  251 ++++
 .../solr/client/solrj/util/package-info.java    |   23 +
 .../org/apache/solr/common/EnumFieldValue.java  |  116 ++
 .../org/apache/solr/common/SolrDocument.java    |  396 ++++++
 .../apache/solr/common/SolrDocumentList.java    |   68 +
 .../org/apache/solr/common/SolrException.java   |  208 +++
 .../apache/solr/common/SolrInputDocument.java   |  301 +++++
 .../org/apache/solr/common/SolrInputField.java  |  232 ++++
 .../org/apache/solr/common/StringUtils.java     |   26 +
 .../org/apache/solr/common/cloud/Aliases.java   |   63 +
 .../solr/common/cloud/BeforeReconnect.java      |   22 +
 .../solr/common/cloud/ClosableThread.java       |   27 +
 .../apache/solr/common/cloud/ClusterState.java  |  397 ++++++
 .../solr/common/cloud/ClusterStateUtil.java     |  230 ++++
 .../solr/common/cloud/CompositeIdRouter.java    |  327 +++++
 .../solr/common/cloud/ConnectionManager.java    |  237 ++++
 .../common/cloud/DefaultConnectionStrategy.java |   75 ++
 .../solr/common/cloud/DefaultZkACLProvider.java |   45 +
 .../cloud/DefaultZkCredentialsProvider.java     |   41 +
 .../apache/solr/common/cloud/DocCollection.java |  201 +++
 .../org/apache/solr/common/cloud/DocRouter.java |  227 ++++
 .../solr/common/cloud/HashBasedRouter.java      |   81 ++
 .../solr/common/cloud/ImplicitDocRouter.java    |  104 ++
 .../apache/solr/common/cloud/OnReconnect.java   |   22 +
 .../apache/solr/common/cloud/PlainIdRouter.java |   23 +
 .../org/apache/solr/common/cloud/Replica.java   |   48 +
 .../apache/solr/common/cloud/RoutingRule.java   |   71 +
 .../solr/common/cloud/SaslZkACLProvider.java    |   49 +
 .../org/apache/solr/common/cloud/Slice.java     |  196 +++
 .../apache/solr/common/cloud/SolrZkClient.java  |  736 +++++++++++
 .../apache/solr/common/cloud/SolrZooKeeper.java |  103 ++
 ...ParamsAllAndReadonlyDigestZkACLProvider.java |   89 ++
 ...tCredentialsDigestZkCredentialsProvider.java |   60 +
 .../apache/solr/common/cloud/ZkACLProvider.java |   28 +
 .../cloud/ZkClientConnectionStrategy.java       |  113 ++
 .../apache/solr/common/cloud/ZkCmdExecutor.java |  111 ++
 .../solr/common/cloud/ZkConfigManager.java      |  145 +++
 .../solr/common/cloud/ZkCoreNodeProps.java      |   74 ++
 .../common/cloud/ZkCredentialsProvider.java     |   45 +
 .../apache/solr/common/cloud/ZkNodeProps.java   |  154 +++
 .../apache/solr/common/cloud/ZkOperation.java   |   37 +
 .../apache/solr/common/cloud/ZkStateReader.java |  925 +++++++++++++
 .../solr/common/cloud/ZooKeeperException.java   |   33 +
 .../apache/solr/common/cloud/package-info.java  |   23 +
 .../org/apache/solr/common/luke/FieldFlag.java  |   70 +
 .../apache/solr/common/luke/package-info.java   |   23 +
 .../org/apache/solr/common/package-info.java    |   23 +
 .../solr/common/params/AnalysisParams.java      |   60 +
 .../solr/common/params/AppendedSolrParams.java  |   55 +
 .../solr/common/params/CollectionParams.java    |   74 ++
 .../apache/solr/common/params/CommonParams.java |  228 ++++
 .../solr/common/params/CoreAdminParams.java     |  151 +++
 .../solr/common/params/CursorMarkParams.java    |   48 +
 .../solr/common/params/DefaultSolrParams.java   |   68 +
 .../apache/solr/common/params/DisMaxParams.java |   78 ++
 .../apache/solr/common/params/EventParams.java  |   29 +
 .../apache/solr/common/params/ExpandParams.java |   32 +
 .../apache/solr/common/params/FacetParams.java  |  405 ++++++
 .../apache/solr/common/params/GroupParams.java  |   71 +
 .../solr/common/params/HighlightParams.java     |   82 ++
 .../solr/common/params/MapSolrParams.java       |   88 ++
 .../common/params/ModifiableSolrParams.java     |  210 +++
 .../solr/common/params/MoreLikeThisParams.java  |   74 ++
 .../solr/common/params/MultiMapSolrParams.java  |   92 ++
 .../common/params/QueryElevationParams.java     |   53 +
 .../solr/common/params/RequiredSolrParams.java  |  155 +++
 .../apache/solr/common/params/ShardParams.java  |   56 +
 .../apache/solr/common/params/SimpleParams.java |   50 +
 .../apache/solr/common/params/SolrParams.java   |  363 ++++++
 .../solr/common/params/SpatialParams.java       |   41 +
 .../solr/common/params/SpellingParams.java      |  174 +++
 .../apache/solr/common/params/StatsParams.java  |   28 +
 .../solr/common/params/TermVectorParams.java    |   66 +
 .../apache/solr/common/params/TermsParams.java  |  120 ++
 .../apache/solr/common/params/UpdateParams.java |   72 +
 .../apache/solr/common/params/package-info.java |   22 +
 .../org/apache/solr/common/util/Base64.java     |  153 +++
 .../org/apache/solr/common/util/ByteUtils.java  |  126 ++
 .../apache/solr/common/util/ContentStream.java  |   81 ++
 .../solr/common/util/ContentStreamBase.java     |  260 ++++
 .../solr/common/util/DataInputInputStream.java  |   27 +
 .../org/apache/solr/common/util/DateUtil.java   |  260 ++++
 .../apache/solr/common/util/ExecutorUtil.java   |   64 +
 .../solr/common/util/FastInputStream.java       |  253 ++++
 .../solr/common/util/FastOutputStream.java      |  233 ++++
 .../java/org/apache/solr/common/util/Hash.java  |  422 ++++++
 .../org/apache/solr/common/util/IOUtils.java    |   37 +
 .../apache/solr/common/util/IteratorChain.java  |   87 ++
 .../apache/solr/common/util/JavaBinCodec.java   |  820 ++++++++++++
 .../solr/common/util/JsonRecordReader.java      |  586 +++++++++
 .../org/apache/solr/common/util/NamedList.java  |  708 ++++++++++
 .../solr/common/util/ObjectReleaseTracker.java  |   62 +
 .../org/apache/solr/common/util/RetryUtil.java  |   43 +
 .../solr/common/util/SimpleOrderedMap.java      |   67 +
 .../common/util/SolrjNamedThreadFactory.java    |   50 +
 .../org/apache/solr/common/util/StrUtils.java   |  309 +++++
 .../org/apache/solr/common/util/URLUtil.java    |   50 +
 .../java/org/apache/solr/common/util/XML.java   |  207 +++
 .../apache/solr/common/util/XMLErrorLogger.java |   84 ++
 .../apache/solr/common/util/package-info.java   |   23 +
 ranger_solrj/src/main/java/overview.html        |   21 +
 security-admin/pom.xml                          |    5 +
 security-admin/scripts/install.properties       |   10 +
 security-admin/scripts/setup.sh                 |   48 +-
 .../java/org/apache/ranger/biz/AssetMgr.java    |   11 +-
 .../org/apache/ranger/biz/RangerBizUtil.java    |  434 +++---
 .../java/org/apache/ranger/biz/XAuditMgr.java   |   41 +-
 .../apache/ranger/db/RangerDaoManagerBase.java  |    3 +
 .../ranger/solr/SolrAccessAuditsService.java    |  253 ++++
 .../java/org/apache/ranger/solr/SolrMgr.java    |   99 ++
 .../java/org/apache/ranger/solr/SolrUtil.java   |  327 +++++
 src/main/assembly/hbase-agent.xml               |    6 +
 src/main/assembly/hdfs-agent.xml                |    6 +
 src/main/assembly/hive-agent.xml                |    6 +
 src/main/assembly/knox-agent.xml                |    6 +
 src/main/assembly/storm-agent.xml               |    6 +
 storm-agent/conf/ranger-storm-audit-changes.cfg |   13 +
 storm-agent/conf/ranger-storm-audit.xml         |   48 +
 storm-agent/scripts/install.properties          |    6 +
 207 files changed, 30674 insertions(+), 219 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/pom.xml
----------------------------------------------------------------------
diff --git a/agents-audit/pom.xml b/agents-audit/pom.xml
index e54ec36..6715575 100644
--- a/agents-audit/pom.xml
+++ b/agents-audit/pom.xml
@@ -63,9 +63,29 @@
 	<version>${log4j.version}</version>
     </dependency>
     <dependency>
-	<groupId>org.apache.kafka</groupId>
-	<artifactId>kafka_2.10</artifactId>
-	<version>0.8.2.0</version>
-</dependency>
+      <groupId>org.apache.kafka</groupId>
+      <artifactId>kafka_2.10</artifactId>
+      <version>${kafka.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.ranger</groupId>
+      <artifactId>ranger_solrj</artifactId>
+      <version>${ranger.solrj.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>${httpcomponent.httpclient.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpmime</artifactId>
+      <version>${httpcomponent.httpmime.version}</version>
+    </dependency>	
+    <dependency>
+      <groupId>org.noggit</groupId>
+      <artifactId>noggit</artifactId>
+      <version>${noggit.version}</version>
+    </dependency>    
   </dependencies>
 </project>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
index 9fbe29f..bb8fa6d 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/AuditProviderFactory.java
@@ -26,6 +26,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.ranger.audit.provider.hdfs.HdfsAuditProvider;
 import org.apache.ranger.audit.provider.kafka.KafkaAuditProvider;
+import org.apache.ranger.audit.provider.solr.SolrAuditProvider;
 
 
 /*
@@ -43,6 +44,7 @@ public class AuditProviderFactory {
 	private static final String AUDIT_HDFS_IS_ENABLED_PROP  = "xasecure.audit.hdfs.is.enabled";
 	private static final String AUDIT_LOG4J_IS_ENABLED_PROP = "xasecure.audit.log4j.is.enabled" ;
 	private static final String AUDIT_KAFKA_IS_ENABLED_PROP = "xasecure.audit.kafka.is.enabled";
+	private static final String AUDIT_SOLR_IS_ENABLED_PROP = "xasecure.audit.solr.is.enabled";
 	
 	private static final int AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT     = 10 * 1024;
 	private static final int AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT =  5 * 1000;
@@ -99,8 +101,9 @@ public class AuditProviderFactory {
 		boolean isAuditToHdfsEnabled  = BaseAuditProvider.getBooleanProperty(props, AUDIT_HDFS_IS_ENABLED_PROP, false);
 		boolean isAuditToLog4jEnabled = BaseAuditProvider.getBooleanProperty(props, AUDIT_LOG4J_IS_ENABLED_PROP, false);
 		boolean isAuditToKafkaEnabled  = BaseAuditProvider.getBooleanProperty(props, AUDIT_KAFKA_IS_ENABLED_PROP, false);
+		boolean isAuditToSolrEnabled  = BaseAuditProvider.getBooleanProperty(props, AUDIT_SOLR_IS_ENABLED_PROP, false);
 
-		if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled)) {
+		if(!isEnabled || !(isAuditToDbEnabled || isAuditToHdfsEnabled || isAuditToKafkaEnabled || isAuditToLog4jEnabled || isAuditToSolrEnabled)) {
 			LOG.info("AuditProviderFactory: Audit not enabled..");
 
 			mProvider = getDefaultProvider();
@@ -111,6 +114,7 @@ public class AuditProviderFactory {
 		List<AuditProvider> providers = new ArrayList<AuditProvider>();
 
 		if(isAuditToDbEnabled) {
+			LOG.info("DbAuditProvider is enabled");
 			DbAuditProvider dbProvider = new DbAuditProvider();
 
 			boolean isAuditToDbAsync = BaseAuditProvider.getBooleanProperty(props, DbAuditProvider.AUDIT_DB_IS_ASYNC_PROP, false);
@@ -128,6 +132,8 @@ public class AuditProviderFactory {
 		}
 
 		if(isAuditToHdfsEnabled) {
+			LOG.info("HdfsAuditProvider is enabled");
+
 			HdfsAuditProvider hdfsProvider = new HdfsAuditProvider();
 
 			boolean isAuditToHdfsAsync = BaseAuditProvider.getBooleanProperty(props, HdfsAuditProvider.AUDIT_HDFS_IS_ASYNC_PROP, false);
@@ -156,7 +162,20 @@ public class AuditProviderFactory {
 				providers.add(kafkaProvider);
 			}
 		}
-		
+
+		if(isAuditToSolrEnabled) {
+			LOG.info("SolrAuditProvider is enabled");
+			SolrAuditProvider solrProvider = new SolrAuditProvider();
+			solrProvider.init(props);
+			
+			if( solrProvider.isAsync()) {
+				AsyncAuditProvider asyncProvider = new AsyncAuditProvider("MySolrAuditProvider", solrProvider.getMaxQueueSize(), solrProvider.getMaxFlushInterval(), solrProvider);
+				providers.add(asyncProvider);
+			} else {
+				providers.add(solrProvider);
+			}
+		}
+
 		if(isAuditToLog4jEnabled) {
 			Log4jAuditProvider log4jProvider = new Log4jAuditProvider();
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
index 14e6220..a068b8f 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/BaseAuditProvider.java
@@ -40,7 +40,7 @@ public abstract class BaseAuditProvider implements AuditProvider {
 	private int maxQueueSize     =  AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT;
 	private int maxFlushInterval = AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT;
 
-	
+	protected Properties props = null;
 
 	public BaseAuditProvider() {
 	}
@@ -48,7 +48,8 @@ public abstract class BaseAuditProvider implements AuditProvider {
 	@Override
 	public void init(Properties props) {
 		LOG.info("BaseAuditProvider.init()");
-
+		this.props = props;
+		
 		mLogFailureReportMinIntervalInMs = getIntProperty(props, AUDIT_LOG_FAILURE_REPORT_MIN_INTERVAL_PROP, 60 * 1000);
 
 	}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
index 54e73ea..0ec8790 100644
--- a/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/kafka/KafkaAuditProvider.java
@@ -117,7 +117,7 @@ public class KafkaAuditProvider extends BaseAuditProvider {
 			}
 		} catch (Throwable t) {
 			LOG.error("Error sending message to Kafka topic. topic=" + topic
-					+ ", message=" + message);
+					+ ", message=" + message, t);
 		}
 	}
 

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
----------------------------------------------------------------------
diff --git a/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
new file mode 100644
index 0000000..1b463e6
--- /dev/null
+++ b/agents-audit/src/main/java/org/apache/ranger/audit/provider/solr/SolrAuditProvider.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.audit.provider.solr;
+
+import java.util.Date;
+import java.util.Properties;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.ranger.audit.model.AuditEventBase;
+import org.apache.ranger.audit.model.AuthzAuditEvent;
+import org.apache.ranger.audit.provider.BaseAuditProvider;
+import org.apache.ranger.audit.provider.MiscUtil;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrInputDocument;
+
+public class SolrAuditProvider extends BaseAuditProvider {
+	private static final Log LOG = LogFactory.getLog(SolrAuditProvider.class);
+
+	public static final String AUDIT_MAX_QUEUE_SIZE_PROP = "xasecure.audit.solr.async.max.queue.size";
+	public static final String AUDIT_MAX_FLUSH_INTERVAL_PROP = "xasecure.audit.solr.async.max.flush.interval.ms";
+	public static final String AUDIT_RETRY_WAIT_PROP = "xasecure.audit.solr.retry.ms";
+
+	static final Object lock = new Object();
+	SolrClient solrClient = null;
+	Date lastConnectTime = null;
+	long lastFailTime = 0;
+
+	int retryWaitTime = 30000;
+
+	public SolrAuditProvider() {
+	}
+
+	@Override
+	public void init(Properties props) {
+		LOG.info("init() called");
+		super.init(props);
+
+		setMaxQueueSize(BaseAuditProvider.getIntProperty(props,
+				AUDIT_MAX_QUEUE_SIZE_PROP, AUDIT_ASYNC_MAX_QUEUE_SIZE_DEFAULT));
+		setMaxFlushInterval(BaseAuditProvider.getIntProperty(props,
+				AUDIT_MAX_QUEUE_SIZE_PROP,
+				AUDIT_ASYNC_MAX_FLUSH_INTERVAL_DEFAULT));
+		retryWaitTime = BaseAuditProvider.getIntProperty(props,
+				AUDIT_RETRY_WAIT_PROP, retryWaitTime);
+	}
+
+	void connect() {
+		if (solrClient == null) {
+			synchronized (lock) {
+				if (solrClient == null) {
+					String solrURL = BaseAuditProvider.getStringProperty(props,
+							"xasecure.audit.solr.solr_url");
+
+					if (lastConnectTime != null) {
+						// Let's wait for enough time before retrying
+						long diff = System.currentTimeMillis()
+								- lastConnectTime.getTime();
+						if (diff < retryWaitTime) {
+							if (LOG.isDebugEnabled()) {
+								LOG.debug("Ignore connecting to solr url="
+										+ solrURL + ", lastConnect=" + diff
+										+ "ms");
+							}
+							return;
+						}
+					}
+					lastConnectTime = new Date();
+
+					if (solrURL == null || solrURL.isEmpty()) {
+						LOG.fatal("Solr URL for Audit is empty");
+						return;
+					}
+
+					try {
+						// TODO: Need to support SolrCloud also
+						solrClient = new HttpSolrClient(solrURL);
+						if (solrClient instanceof HttpSolrClient) {
+							HttpSolrClient httpSolrClient = (HttpSolrClient) solrClient;
+							httpSolrClient.setAllowCompression(true);
+							httpSolrClient.setConnectionTimeout(1000);
+							// solrClient.setSoTimeout(10000);
+							httpSolrClient.setMaxRetries(1);
+						}
+					} catch (Throwable t) {
+						LOG.fatal("Can't connect to Solr server. URL="
+								+ solrURL, t);
+					}
+				}
+			}
+		}
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see
+	 * org.apache.ranger.audit.provider.AuditProvider#log(org.apache.ranger.
+	 * audit.model.AuditEventBase)
+	 */
+	@Override
+	public void log(AuditEventBase event) {
+		if (!(event instanceof AuthzAuditEvent)) {
+			LOG.error(event.getClass().getName()
+					+ " audit event class type is not supported");
+			return;
+		}
+		AuthzAuditEvent authzEvent = (AuthzAuditEvent) event;
+		// TODO: This should be done at a higher level
+
+		if (authzEvent.getAgentHostname() == null) {
+			authzEvent.setAgentHostname(MiscUtil.getHostname());
+		}
+
+		if (authzEvent.getLogType() == null) {
+			authzEvent.setLogType("RangerAudit");
+		}
+
+		if (authzEvent.getEventId() == null) {
+			authzEvent.setEventId(MiscUtil.generateUniqueId());
+		}
+
+		try {
+			if (solrClient == null) {
+				connect();
+				if (solrClient == null) {
+					// Solr is still not initialized. So need to throw error
+					return;
+				}
+			}
+
+			if (lastFailTime > 0) {
+				long diff = System.currentTimeMillis() - lastFailTime;
+				if (diff < retryWaitTime) {
+					if (LOG.isDebugEnabled()) {
+						LOG.debug("Ignore sending audit. lastConnect=" + diff
+								+ " ms");
+					}
+					return;
+				}
+			}
+			// Convert AuditEventBase to Solr document
+			SolrInputDocument document = toSolrDoc(authzEvent);
+			UpdateResponse response = solrClient.add(document);
+			if (response.getStatus() != 0) {
+				lastFailTime = System.currentTimeMillis();
+
+				// System.out.println("Response=" + response.toString()
+				// + ", status= " + response.getStatus() + ", event="
+				// + event);
+				// throw new Exception("Aborting. event=" + event +
+				// ", response="
+				// + response.toString());
+			} else {
+				lastFailTime = 0;
+			}
+
+		} catch (Throwable t) {
+			LOG.error("Error sending message to Solr", t);
+		}
+
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see org.apache.ranger.audit.provider.AuditProvider#start()
+	 */
+	@Override
+	public void start() {
+		connect();
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see org.apache.ranger.audit.provider.AuditProvider#stop()
+	 */
+	@Override
+	public void stop() {
+		// TODO Auto-generated method stub
+
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see org.apache.ranger.audit.provider.AuditProvider#waitToComplete()
+	 */
+	@Override
+	public void waitToComplete() {
+		// TODO Auto-generated method stub
+
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see org.apache.ranger.audit.provider.AuditProvider#isFlushPending()
+	 */
+	@Override
+	public boolean isFlushPending() {
+		// TODO Auto-generated method stub
+		return false;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see org.apache.ranger.audit.provider.AuditProvider#getLastFlushTime()
+	 */
+	@Override
+	public long getLastFlushTime() {
+		// TODO Auto-generated method stub
+		return 0;
+	}
+
+	/*
+	 * (non-Javadoc)
+	 * 
+	 * @see org.apache.ranger.audit.provider.AuditProvider#flush()
+	 */
+	@Override
+	public void flush() {
+		// TODO Auto-generated method stub
+
+	}
+
+	SolrInputDocument toSolrDoc(AuthzAuditEvent auditEvent) {
+		SolrInputDocument doc = new SolrInputDocument();
+		doc.addField("id", auditEvent.getEventId());
+		doc.addField("access", auditEvent.getAccessType());
+		doc.addField("enforcer", auditEvent.getAclEnforcer());
+		doc.addField("agent", auditEvent.getAgentId());
+		doc.addField("repo", auditEvent.getRepositoryName());
+		doc.addField("sess", auditEvent.getSessionId());
+		doc.addField("reqUser", auditEvent.getUser());
+		doc.addField("reqData", auditEvent.getRequestData());
+		doc.addField("resource", auditEvent.getResourcePath());
+		doc.addField("cliIP", auditEvent.getClientIP());
+		doc.addField("logType", auditEvent.getLogType());
+		doc.addField("result", auditEvent.getAccessResult());
+		doc.addField("policy", auditEvent.getPolicyId());
+		doc.addField("repoType", auditEvent.getRepositoryType());
+		doc.addField("resType", auditEvent.getResourceType());
+		doc.addField("reason", auditEvent.getResultReason());
+		doc.addField("action", auditEvent.getAction());
+		doc.addField("evtTime", auditEvent.getEventTime());
+		return doc;
+	}
+	
+	public boolean isAsync() {
+		return true;
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/ranger-hbase-audit-changes.cfg b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
index e5c381a..221d20a 100644
--- a/hbase-agent/conf/ranger-hbase-audit-changes.cfg
+++ b/hbase-agent/conf/ranger-hbase-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%   mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directory                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%             mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled                                    %XAAUDIT.KAFKA.IS_ENABLED%                             mod create-if-not-exists
+#xasecure.audit.kafka.is.async                                      %XAAUDIT.KAFKA.IS_ASYNC%                               mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size                          %XAAUDIT.KAFKA.MAX_QUEUE_SIZE%                         mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms                   %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS%                  mod create-if-not-exists
+#xasecure.audit.kafka.broker_list                                   %XAAUDIT.KAFKA.BROKER_LIST%                            mod create-if-not-exists
+#xasecure.audit.kafka.topic_name                                    %XAAUDIT.KAFKA.TOPIC_NAME%                             mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled                                    %XAAUDIT.SOLR.IS_ENABLED%                               mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size                          %XAAUDIT.SOLR.MAX_QUEUE_SIZE%                           mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms                   %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS%                    mod create-if-not-exists
+xasecure.audit.solr.solr_url                                      %XAAUDIT.SOLR.SOLR_URL%                                 mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/conf/ranger-hbase-audit.xml
----------------------------------------------------------------------
diff --git a/hbase-agent/conf/ranger-hbase-audit.xml b/hbase-agent/conf/ranger-hbase-audit.xml
index b39696b..e5bfb89 100644
--- a/hbase-agent/conf/ranger-hbase-audit.xml
+++ b/hbase-agent/conf/ranger-hbase-audit.xml
@@ -183,4 +183,52 @@
 		<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
 		<value>30000</value>
 	</property>	
+	
+	<!-- Kafka audit provider configuration -->
+	<property>
+		<name>xasecure.audit.kafka.is.enabled</name>
+		<value>false</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.kafka.broker_list</name>
+		<value>localhost:9092</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.topic_name</name>
+		<value>ranger_audits</value>
+	</property>	
+	
+	<!-- Ranger audit provider configuration -->
+	<property>
+		<name>xasecure.audit.ranger.is.enabled</name>
+		<value>false</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.ranger.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.solr.solr_url</name>
+		<value>http://localhost:6083/solr/ranger_audits</value>
+	</property>	
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hbase-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hbase-agent/scripts/install.properties b/hbase-agent/scripts/install.properties
index 5a81ad4..7ff29c9 100644
--- a/hbase-agent/scripts/install.properties
+++ b/hbase-agent/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
 
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
 #
 # SSL Client Certificate Information
 #

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
index e5c381a..8d31016 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
+++ b/hdfs-agent/conf/ranger-hdfs-audit-changes.cfg
@@ -31,3 +31,14 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%   mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directory                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%             mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled                                    %XAAUDIT.KAFKA.IS_ENABLED%                             mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size                          %XAAUDIT.KAFKA.MAX_QUEUE_SIZE%                         mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms                   %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS%                  mod create-if-not-exists
+#xasecure.audit.kafka.broker_list                                   %XAAUDIT.KAFKA.BROKER_LIST%                            mod create-if-not-exists
+#xasecure.audit.kafka.topic_name                                    %XAAUDIT.KAFKA.TOPIC_NAME%                             mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled                                    %XAAUDIT.SOLR.IS_ENABLED%                               mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size                          %XAAUDIT.SOLR.MAX_QUEUE_SIZE%                           mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms                   %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS%                    mod create-if-not-exists
+xasecure.audit.solr.solr_url                                      %XAAUDIT.SOLR.SOLR_URL%                                 mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/conf/ranger-hdfs-audit.xml
----------------------------------------------------------------------
diff --git a/hdfs-agent/conf/ranger-hdfs-audit.xml b/hdfs-agent/conf/ranger-hdfs-audit.xml
index d26345d..09114ad 100644
--- a/hdfs-agent/conf/ranger-hdfs-audit.xml
+++ b/hdfs-agent/conf/ranger-hdfs-audit.xml
@@ -183,4 +183,52 @@
 		<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
 		<value>30000</value>
 	</property>	
+	
+	<!-- Kafka audit provider configuration -->
+	<property>
+		<name>xasecure.audit.kafka.is.enabled</name>
+		<value>false</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.kafka.broker_list</name>
+		<value>localhost:9092</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.topic_name</name>
+		<value>ranger_audits</value>
+	</property>	
+	
+	<!-- Ranger audit provider configuration -->
+	<property>
+		<name>xasecure.audit.ranger.is.enabled</name>
+		<value>false</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.ranger.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.solr.solr_url</name>
+		<value>http://localhost:6083/solr/ranger_audits</value>
+	</property>	
+	
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hdfs-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hdfs-agent/scripts/install.properties b/hdfs-agent/scripts/install.properties
index 93790e3..2e1b61a 100644
--- a/hdfs-agent/scripts/install.properties
+++ b/hdfs-agent/scripts/install.properties
@@ -89,6 +89,13 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
 
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
+
 #
 # SSL Client Certificate Information
 #

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/hive-agent/conf/ranger-hive-audit-changes.cfg b/hive-agent/conf/ranger-hive-audit-changes.cfg
index 9fa7608..2d6d414 100644
--- a/hive-agent/conf/ranger-hive-audit-changes.cfg
+++ b/hive-agent/conf/ranger-hive-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%   mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directory                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%             mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled                                    %XAAUDIT.KAFKA.IS_ENABLED%                             mod create-if-not-exists
+#xasecure.audit.kafka.is.async                                      %XAAUDIT.KAFKA.IS_ASYNC%                               mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size                          %XAAUDIT.KAFKA.MAX_QUEUE_SIZE%                         mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms                   %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS%                  mod create-if-not-exists
+#xasecure.audit.kafka.broker_list                                   %XAAUDIT.KAFKA.BROKER_LIST%                            mod create-if-not-exists
+#xasecure.audit.kafka.topic_name                                    %XAAUDIT.KAFKA.TOPIC_NAME%                             mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled                                    %XAAUDIT.SOLR.IS_ENABLED%                               mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size                          %XAAUDIT.SOLR.MAX_QUEUE_SIZE%                           mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms                   %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS%                    mod create-if-not-exists
+xasecure.audit.solr.solr_url                                      %XAAUDIT.SOLR.SOLR_URL%                                 mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/conf/ranger-hive-audit.xml
----------------------------------------------------------------------
diff --git a/hive-agent/conf/ranger-hive-audit.xml b/hive-agent/conf/ranger-hive-audit.xml
index d011b24..e753336 100644
--- a/hive-agent/conf/ranger-hive-audit.xml
+++ b/hive-agent/conf/ranger-hive-audit.xml
@@ -183,4 +183,52 @@
 		<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
 		<value>30000</value>
 	</property>	
+	
+	<!-- Kafka audit provider configuration -->
+	<property>
+		<name>xasecure.audit.kafka.is.enabled</name>
+		<value>false</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.kafka.broker_list</name>
+		<value>localhost:9092</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.topic_name</name>
+		<value>ranger_audits</value>
+	</property>	
+	
+	<!-- Ranger audit provider configuration -->
+	<property>
+		<name>xasecure.audit.ranger.is.enabled</name>
+		<value>false</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.ranger.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.solr.solr_url</name>
+		<value>http://localhost:6083/solr/ranger_audits</value>
+	</property>	
+	
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/hive-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/hive-agent/scripts/install.properties b/hive-agent/scripts/install.properties
index bbd1849..75b1b5d 100644
--- a/hive-agent/scripts/install.properties
+++ b/hive-agent/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
 
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
 #
 # SSL Client Certificate Information
 #

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/knox-agent/conf/ranger-knox-audit-changes.cfg b/knox-agent/conf/ranger-knox-audit-changes.cfg
index 7ae334e..f97d10f 100644
--- a/knox-agent/conf/ranger-knox-audit-changes.cfg
+++ b/knox-agent/conf/ranger-knox-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%   mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directory                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%             mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled                                    %XAAUDIT.KAFKA.IS_ENABLED%                             mod create-if-not-exists
+#xasecure.audit.kafka.is.async                                      %XAAUDIT.KAFKA.IS_ASYNC%                               mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size                          %XAAUDIT.KAFKA.MAX_QUEUE_SIZE%                         mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms                   %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS%                  mod create-if-not-exists
+#xasecure.audit.kafka.broker_list                                   %XAAUDIT.KAFKA.BROKER_LIST%                            mod create-if-not-exists
+#xasecure.audit.kafka.topic_name                                    %XAAUDIT.KAFKA.TOPIC_NAME%                             mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled                                    %XAAUDIT.SOLR.IS_ENABLED%                               mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size                          %XAAUDIT.SOLR.MAX_QUEUE_SIZE%                           mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms                   %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS%                    mod create-if-not-exists
+xasecure.audit.solr.solr_url                                      %XAAUDIT.SOLR.SOLR_URL%                                 mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/conf/ranger-knox-audit.xml
----------------------------------------------------------------------
diff --git a/knox-agent/conf/ranger-knox-audit.xml b/knox-agent/conf/ranger-knox-audit.xml
index 0fdcefc..6f0adb9 100644
--- a/knox-agent/conf/ranger-knox-audit.xml
+++ b/knox-agent/conf/ranger-knox-audit.xml
@@ -183,4 +183,52 @@
 		<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
 		<value>30000</value>
 	</property>	
+	
+	<!-- Kafka audit provider configuration -->
+	<property>
+		<name>xasecure.audit.kafka.is.enabled</name>
+		<value>false</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.kafka.broker_list</name>
+		<value>localhost:9092</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.topic_name</name>
+		<value>ranger_audits</value>
+	</property>	
+	
+	<!-- Ranger audit provider configuration -->
+	<property>
+		<name>xasecure.audit.ranger.is.enabled</name>
+		<value>false</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.ranger.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.solr.solr_url</name>
+		<value>http://localhost:6083/solr/ranger_audits</value>
+	</property>	
+	
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/knox-agent/scripts/install.properties
----------------------------------------------------------------------
diff --git a/knox-agent/scripts/install.properties b/knox-agent/scripts/install.properties
index d821c5d..ecd9813 100644
--- a/knox-agent/scripts/install.properties
+++ b/knox-agent/scripts/install.properties
@@ -92,6 +92,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
 
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
 #
 # SSL Client Certificate Information
 #

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
----------------------------------------------------------------------
diff --git a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
index 4f2c5a2..e0dbea2 100644
--- a/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
+++ b/plugin-yarn/conf/ranger-yarn-audit-changes.cfg
@@ -31,3 +31,15 @@ xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds     %XAAUDIT.HDFS
 xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds  %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS%   mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.directory                 %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY%                  mod create-if-not-exists
 xasecure.audit.hdfs.config.local.archive.max.file.count            %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT%             mod create-if-not-exists
+
+#xasecure.audit.kafka.is.enabled                                    %XAAUDIT.KAFKA.IS_ENABLED%                             mod create-if-not-exists
+#xasecure.audit.kafka.is.async                                      %XAAUDIT.KAFKA.IS_ASYNC%                               mod create-if-not-exists
+#xasecure.audit.kafka.async.max.queue.size                          %XAAUDIT.KAFKA.MAX_QUEUE_SIZE%                         mod create-if-not-exists
+#xasecure.audit.kafka.async.max.flush.interval.ms                   %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS%                  mod create-if-not-exists
+#xasecure.audit.kafka.broker_list                                   %XAAUDIT.KAFKA.BROKER_LIST%                            mod create-if-not-exists
+#xasecure.audit.kafka.topic_name                                    %XAAUDIT.KAFKA.TOPIC_NAME%                             mod create-if-not-exists
+
+xasecure.audit.solr.is.enabled                                    %XAAUDIT.SOLR.IS_ENABLED%                               mod create-if-not-exists
+xasecure.audit.solr.async.max.queue.size                          %XAAUDIT.SOLR.MAX_QUEUE_SIZE%                           mod create-if-not-exists
+xasecure.audit.solr.async.max.flush.interval.ms                   %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS%                    mod create-if-not-exists
+xasecure.audit.solr.solr_url                                      %XAAUDIT.SOLR.SOLR_URL%                                 mod create-if-not-exists

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/conf/ranger-yarn-audit.xml
----------------------------------------------------------------------
diff --git a/plugin-yarn/conf/ranger-yarn-audit.xml b/plugin-yarn/conf/ranger-yarn-audit.xml
index c0096a4..f1e9687 100644
--- a/plugin-yarn/conf/ranger-yarn-audit.xml
+++ b/plugin-yarn/conf/ranger-yarn-audit.xml
@@ -184,4 +184,52 @@
 		<name>xasecure.audit.log4j.async.max.flush.interval.ms</name>
 		<value>30000</value>
 	</property>	
+	
+	<!-- Kafka audit provider configuration -->
+	<property>
+		<name>xasecure.audit.kafka.is.enabled</name>
+		<value>false</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.kafka.broker_list</name>
+		<value>localhost:9092</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.kafka.topic_name</name>
+		<value>ranger_audits</value>
+	</property>	
+	
+	<!-- Ranger audit provider configuration -->
+	<property>
+		<name>xasecure.audit.ranger.is.enabled</name>
+		<value>false</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.ranger.async.max.queue.size</name>
+		<value>1</value>
+	</property>	
+
+	<property>
+		<name>xasecure.audit.ranger.async.max.flush.interval.ms</name>
+		<value>1000</value>
+	</property>	
+	
+	<property>
+		<name>xasecure.audit.solr.solr_url</name>
+		<value>http://localhost:6083/solr/ranger_audits</value>
+	</property>	
+
 </configuration>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/plugin-yarn/scripts/install.properties
----------------------------------------------------------------------
diff --git a/plugin-yarn/scripts/install.properties b/plugin-yarn/scripts/install.properties
index d2d1ffe..bbe9f7f 100644
--- a/plugin-yarn/scripts/install.properties
+++ b/plugin-yarn/scripts/install.properties
@@ -89,6 +89,12 @@ XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60
 XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600
 XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10
 
+#Solr Audit Provder
+XAAUDIT.SOLR.IS_ENABLED=false
+XAAUDIT.SOLR.MAX_QUEUE_SIZE=1
+XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000
+XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits
+
 #
 # SSL Client Certificate Information
 #

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ef39d37..00c8339 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,6 +88,7 @@
   <module>knox-agent</module>
   <module>storm-agent</module>
   <module>plugin-yarn</module>
+  <module>ranger_solrj</module>
   <module>security-admin</module>
   <module>ugsync</module>
   <module>unixauthclient</module>
@@ -129,6 +130,9 @@
 		<hamcrest.all.version>1.3</hamcrest.all.version>
 		<hbase.version>0.99.2</hbase.version>
 		<hive.version>1.2.0-SNAPSHOT</hive.version>
+		<httpcomponent.httpmime.version>4.2.5</httpcomponent.httpmime.version>
+		<httpcomponent.httpclient.version>4.2.5</httpcomponent.httpclient.version>
+		<httpcomponent.httpcore.version>4.2.5</httpcomponent.httpcore.version>
 		<calcite.version>0.9.2-incubating</calcite.version>
 		<tez.version>0.5.2</tez.version>
 		<javassist.version>3.12.1.GA</javassist.version>
@@ -138,16 +142,20 @@
 		<jersey-bundle.version>1.17.1</jersey-bundle.version>
 		<jersey-client.version>2.6</jersey-client.version>
 		<junit.version>4.11</junit.version>
+		<kafka.version>0.8.2.0</kafka.version>
 		<mockito.version>1.8.4</mockito.version>
 		<hamcrest-version>1.3</hamcrest-version>
 		<knox.gateway.version>0.5.0</knox.gateway.version>
 		<local.lib.dir>${project.basedir}/../lib/local</local.lib.dir>
 		<log4j.version>1.2.17</log4j.version>
 		<mysql-connector-java.version>5.1.31</mysql-connector-java.version>
+		<noggit.version>0.6</noggit.version>
 		<owasp-java-html-sanitizer.version>r239</owasp-java-html-sanitizer.version>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<security-agent-install-dir>hadoop-security/plugins</security-agent-install-dir>
 		<slf4j-api.version>1.7.5</slf4j-api.version>
+		<!--<solr.version>5.0.0</solr.version>-->
+		<ranger.solrj.version>0.4.0</ranger.solrj.version>
 		<springframework.spring.version>2.5.6</springframework.spring.version>
 		<!--
 		<springframework.spring.version>3.1.3.RELEASE</springframework.spring.version>
@@ -162,6 +170,7 @@
 		<tomcat.commons.el.version>5.5.23</tomcat.commons.el.version>
 		<tomcat.embed.version>7.0.55</tomcat.embed.version>
 		<velocity.version>1.7</velocity.version>
+		<zookeeper.version>3.4.6</zookeeper.version>
 		<powermock.version>1.5.6</powermock.version>
 		<aspectj.version>1.8.2</aspectj.version>
 		<findbugs.plugin.version>3.0.0</findbugs.plugin.version>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/.gitignore
----------------------------------------------------------------------
diff --git a/ranger_solrj/.gitignore b/ranger_solrj/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/ranger_solrj/.gitignore
@@ -0,0 +1 @@
+/target/

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/pom.xml
----------------------------------------------------------------------
diff --git a/ranger_solrj/pom.xml b/ranger_solrj/pom.xml
new file mode 100644
index 0000000..2b86140
--- /dev/null
+++ b/ranger_solrj/pom.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <parent>
+    <groupId>org.apache.ranger</groupId>
+    <artifactId>ranger</artifactId>
+    <version>0.4.0</version>
+  </parent>
+  <groupId>org.apache.ranger</groupId>
+  <artifactId>ranger_solrj</artifactId>
+  <version>0.4.0</version>
+  <name>ranger_solrj</name>
+  <url>http://maven.apache.org</url>
+  <properties>
+    <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+  </properties>
+  <dependencies>
+    <dependency>
+      <groupId>commons-io</groupId>
+      <artifactId>commons-io</artifactId>
+      <version>${commons.io.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>${httpcomponent.httpclient.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+      <version>${httpcomponent.httpcore.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpmime</artifactId>
+      <version>${httpcomponent.httpmime.version}</version>
+    </dependency>	
+    <dependency>
+      <groupId>org.noggit</groupId>
+      <artifactId>noggit</artifactId>
+      <version>${noggit.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j-api.version}</version>
+    </dependency>   
+    <dependency>
+      <groupId>org.apache.zookeeper</groupId>
+      <artifactId>zookeeper</artifactId>
+      <version>${zookeeper.version}</version>
+    </dependency>
+  </dependencies>
+</project>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
new file mode 100644
index 0000000..d5c3b38
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/ResponseParser.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import java.io.Reader;
+import java.io.InputStream;
+import org.apache.solr.common.util.NamedList;
+
+/**
+ * 
+ *
+ * @since solr 1.3
+ */
+public abstract class ResponseParser
+{
+  public abstract String getWriterType(); // for example: wt=XML, JSON, etc
+
+  public abstract NamedList<Object> processResponse(InputStream body, String encoding);
+
+  public abstract NamedList<Object> processResponse(Reader reader);
+  
+  /**
+   * A well behaved ResponseParser will return its content-type.
+   * 
+   * @return the content-type this parser expects to parse
+   */
+  public String getContentType() {
+    return null;
+  }
+  
+  /**
+   * @return the version param passed to solr
+   */
+  public String getVersion()
+  {
+    return "2.2";
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/40aa090d/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
----------------------------------------------------------------------
diff --git a/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
new file mode 100644
index 0000000..28b7f4f
--- /dev/null
+++ b/ranger_solrj/src/main/java/org/apache/solr/client/solrj/SolrClient.java
@@ -0,0 +1,416 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj;
+
+import org.apache.solr.client.solrj.SolrRequest.METHOD;
+import org.apache.solr.client.solrj.beans.DocumentObjectBinder;
+import org.apache.solr.client.solrj.impl.StreamingBinaryResponseParser;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.SolrPing;
+import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.client.solrj.response.SolrPingResponse;
+import org.apache.solr.client.solrj.response.UpdateResponse;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.StringUtils;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+
+/**
+ * Abstraction through which all communication with a Solr server may be routed
+ *
+ * @since 5.0, replaced {@code SolrServer}
+ */
+public abstract class SolrClient implements Serializable, Closeable {
+
+  private static final long serialVersionUID = 1L;
+  private DocumentObjectBinder binder;
+
+  /**
+   * Adds a collection of documents
+   * @param docs  the collection of documents
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse add(Collection<SolrInputDocument> docs) throws SolrServerException, IOException {
+    return add(docs, -1);
+  }
+
+  /**
+   * Adds a collection of documents, specifying max time before they become committed
+   * @param docs  the collection of documents
+   * @param commitWithinMs  max time (in ms) before a commit will happen 
+   * @throws IOException If there is a low-level I/O error.
+   * @since solr 3.5
+   */
+  public UpdateResponse add(Collection<SolrInputDocument> docs, int commitWithinMs) throws SolrServerException, IOException {
+    UpdateRequest req = new UpdateRequest();
+    req.add(docs);
+    req.setCommitWithin(commitWithinMs);
+    return req.process(this);
+  }
+
+  /**
+   * Adds a collection of beans
+   * @param beans  the collection of beans
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse addBeans(Collection<?> beans) throws SolrServerException, IOException {
+    return addBeans(beans, -1);
+  }
+
+  /**
+   * Adds a collection of beans specifying max time before they become committed
+   * @param beans  the collection of beans
+   * @param commitWithinMs  max time (in ms) before a commit will happen 
+   * @throws IOException If there is a low-level I/O error.
+   * @since solr 3.5
+   */
+  public UpdateResponse addBeans(Collection<?> beans, int commitWithinMs) throws SolrServerException, IOException {
+    DocumentObjectBinder binder = this.getBinder();
+    ArrayList<SolrInputDocument> docs =  new ArrayList<>(beans.size());
+    for (Object bean : beans) {
+      docs.add(binder.toSolrInputDocument(bean));
+    }
+    return add(docs, commitWithinMs);
+  }
+
+  /**
+   * Adds a single document
+   * @param doc  the input document
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse add(SolrInputDocument doc) throws SolrServerException, IOException {
+    return add(doc, -1);
+  }
+
+  /**
+   * Adds a single document specifying max time before it becomes committed
+   * @param doc  the input document
+   * @param commitWithinMs  max time (in ms) before a commit will happen 
+   * @throws IOException If there is a low-level I/O error.
+   * @since solr 3.5
+   */
+  public UpdateResponse add(SolrInputDocument doc, int commitWithinMs) throws SolrServerException, IOException {
+    UpdateRequest req = new UpdateRequest();
+    req.add(doc);
+    req.setCommitWithin(commitWithinMs);
+    return req.process(this);
+  }
+
+  /**
+   * Adds a single bean
+   * @param obj  the input bean
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse addBean(Object obj) throws IOException, SolrServerException {
+    return addBean(obj, -1);
+  }
+
+  /**
+   * Adds a single bean specifying max time before it becomes committed
+   * @param obj  the input bean
+   * @param commitWithinMs  max time (in ms) before a commit will happen 
+   * @throws IOException If there is a low-level I/O error.
+   * @since solr 3.5
+   */
+  public UpdateResponse addBean(Object obj, int commitWithinMs) throws IOException, SolrServerException {
+    return add(getBinder().toSolrInputDocument(obj),commitWithinMs);
+  }
+
+  /**
+   * Performs an explicit commit, causing pending documents to be committed for indexing
+   * <p>
+   * waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse commit() throws SolrServerException, IOException {
+    return commit(true, true);
+  }
+
+  /**
+   * Performs an explicit optimize, causing a merge of all segments to one.
+   * <p>
+   * waitFlush=true and waitSearcher=true to be inline with the defaults for plain HTTP access
+   * <p>
+   * Note: In most cases it is not required to do explicit optimize
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse optimize() throws SolrServerException, IOException {
+    return optimize(true, true, 1);
+  }
+
+  /**
+   * Performs an explicit commit, causing pending documents to be committed for indexing
+   * @param waitFlush  block until index changes are flushed to disk
+   * @param waitSearcher  block until a new searcher is opened and registered as the main query searcher, making the changes visible 
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse commit(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
+    return new UpdateRequest().setAction(UpdateRequest.ACTION.COMMIT, waitFlush, waitSearcher).process( this );
+  }
+
+  /**
+   * Performs an explicit commit, causing pending documents to be committed for indexing
+   * @param waitFlush  block until index changes are flushed to disk
+   * @param waitSearcher  block until a new searcher is opened and registered as the main query searcher, making the changes visible
+   * @param softCommit makes index changes visible while neither fsync-ing index files nor writing a new index descriptor
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse commit(boolean waitFlush, boolean waitSearcher, boolean softCommit) throws SolrServerException, IOException {
+    return new UpdateRequest().setAction(UpdateRequest.ACTION.COMMIT, waitFlush, waitSearcher, softCommit).process( this );
+  }
+
+  /**
+   * Performs an explicit optimize, causing a merge of all segments to one.
+   * <p>
+   * Note: In most cases it is not required to do explicit optimize
+   * @param waitFlush  block until index changes are flushed to disk
+   * @param waitSearcher  block until a new searcher is opened and registered as the main query searcher, making the changes visible 
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher) throws SolrServerException, IOException {
+    return optimize(waitFlush, waitSearcher, 1);
+  }
+
+  /**
+   * Performs an explicit optimize, causing a merge of all segments to one.
+   * <p>
+   * Note: In most cases it is not required to do explicit optimize
+   * @param waitFlush  block until index changes are flushed to disk
+   * @param waitSearcher  block until a new searcher is opened and registered as the main query searcher, making the changes visible 
+   * @param maxSegments  optimizes down to at most this number of segments
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse optimize(boolean waitFlush, boolean waitSearcher, int maxSegments) throws SolrServerException, IOException {
+    return new UpdateRequest().setAction(UpdateRequest.ACTION.OPTIMIZE, waitFlush, waitSearcher, maxSegments).process( this );
+  }
+
+  /**
+   * Performs a rollback of all non-committed documents pending.
+   * <p>
+   * Note that this is not a true rollback as in databases. Content you have previously
+   * added may have been committed due to autoCommit, buffer full, other client performing
+   * a commit etc.
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse rollback() throws SolrServerException, IOException {
+    return new UpdateRequest().rollback().process( this );
+  }
+
+  /**
+   * Deletes a single document by unique ID
+   * @param id  the ID of the document to delete
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse deleteById(String id) throws SolrServerException, IOException {
+    return deleteById(id, -1);
+  }
+
+  /**
+   * Deletes a single document by unique ID, specifying max time before commit
+   * @param id  the ID of the document to delete
+   * @param commitWithinMs  max time (in ms) before a commit will happen 
+   * @throws IOException If there is a low-level I/O error.
+   * @since 3.6
+   */
+  public UpdateResponse deleteById(String id, int commitWithinMs) throws SolrServerException, IOException {
+    UpdateRequest req = new UpdateRequest();
+    req.deleteById(id);
+    req.setCommitWithin(commitWithinMs);
+    return req.process(this);
+  }
+
+  /**
+   * Deletes a list of documents by unique ID
+   * @param ids  the list of document IDs to delete 
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse deleteById(List<String> ids) throws SolrServerException, IOException {
+    return deleteById(ids, -1);
+  }
+
+  /**
+   * Deletes a list of documents by unique ID, specifying max time before commit
+   * @param ids  the list of document IDs to delete 
+   * @param commitWithinMs  max time (in ms) before a commit will happen 
+   * @throws IOException If there is a low-level I/O error.
+   * @since 3.6
+   */
+  public UpdateResponse deleteById(List<String> ids, int commitWithinMs) throws SolrServerException, IOException {
+    UpdateRequest req = new UpdateRequest();
+    req.deleteById(ids);
+    req.setCommitWithin(commitWithinMs);
+    return req.process(this);
+  }
+
+  /**
+   * Deletes documents from the index based on a query
+   * @param query  the query expressing what documents to delete
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public UpdateResponse deleteByQuery(String query) throws SolrServerException, IOException {
+    return deleteByQuery(query, -1);
+  }
+
+  /**
+   * Deletes documents from the index based on a query, specifying max time before commit
+   * @param query  the query expressing what documents to delete
+   * @param commitWithinMs  max time (in ms) before a commit will happen 
+   * @throws IOException If there is a low-level I/O error.
+   * @since 3.6
+   */
+  public UpdateResponse deleteByQuery(String query, int commitWithinMs) throws SolrServerException, IOException {
+    UpdateRequest req = new UpdateRequest();
+    req.deleteByQuery(query);
+    req.setCommitWithin(commitWithinMs);
+    return req.process(this);
+  }
+
+  /**
+   * Issues a ping request to check if the server is alive
+   * @throws IOException If there is a low-level I/O error.
+   */
+  public SolrPingResponse ping() throws SolrServerException, IOException {
+    return new SolrPing().process(this);
+  }
+
+  /**
+   * Performs a query to the Solr server
+   * @param params  an object holding all key/value parameters to send along the request
+   */
+  public QueryResponse query(SolrParams params) throws SolrServerException, IOException {
+    return new QueryRequest(params).process(this);
+  }
+
+  /**
+   * Performs a query to the Solr server
+   * @param params  an object holding all key/value parameters to send along the request
+   * @param method  specifies the HTTP method to use for the request, such as GET or POST
+   */
+  public QueryResponse query(SolrParams params, METHOD method) throws SolrServerException, IOException {
+    return new QueryRequest(params, method).process(this);
+  }
+
+  /**
+   * Query solr, and stream the results.  Unlike the standard query, this will 
+   * send events for each Document rather then add them to the QueryResponse.
+   *
+   * Although this function returns a 'QueryResponse' it should be used with care
+   * since it excludes anything that was passed to callback.  Also note that
+   * future version may pass even more info to the callback and may not return 
+   * the results in the QueryResponse.
+   *
+   * @since solr 4.0
+   */
+  public QueryResponse queryAndStreamResponse(SolrParams params, StreamingResponseCallback callback) throws SolrServerException, IOException
+  {
+    ResponseParser parser = new StreamingBinaryResponseParser(callback);
+    QueryRequest req = new QueryRequest(params);
+    req.setStreamingResponseCallback(callback);
+    req.setResponseParser(parser);
+    return req.process(this);
+  }
+
+  /**
+   * Retrieves the SolrDocument associated with the given identifier.
+   *
+   * @return retrieved SolrDocument, null if no document is found.
+   */
+  public SolrDocument getById(String id) throws SolrServerException, IOException {
+    return getById(id, null);
+  }
+
+  /**
+   * Retrieves the SolrDocument associated with the given identifier and uses
+   * the SolrParams to execute the request.
+   *
+   * @return retrieved SolrDocument, null if no document is found.
+   */
+  public SolrDocument getById(String id, SolrParams params) throws SolrServerException, IOException {
+    SolrDocumentList docs = getById(Arrays.asList(id), params);
+    if (!docs.isEmpty()) {
+      return docs.get(0);
+    }
+    return null;
+  }
+
+  /**
+   * Retrieves the SolrDocuments associated with the given identifiers.
+   * If a document was not found, it will not be added to the SolrDocumentList.
+   */
+  public SolrDocumentList getById(Collection<String> ids) throws SolrServerException, IOException {
+    return getById(ids, null);
+  }
+
+  /**
+   * Retrieves the SolrDocuments associated with the given identifiers and uses
+   * the SolrParams to execute the request.
+   * If a document was not found, it will not be added to the SolrDocumentList.
+   */
+  public SolrDocumentList getById(Collection<String> ids, SolrParams params) throws SolrServerException, IOException {
+    if (ids == null || ids.isEmpty()) {
+      throw new IllegalArgumentException("Must provide an identifier of a document to retrieve.");
+    }
+
+    ModifiableSolrParams reqParams = new ModifiableSolrParams(params);
+    if (StringUtils.isEmpty(reqParams.get(CommonParams.QT))) {
+      reqParams.set(CommonParams.QT, "/get");
+    }
+    reqParams.set("ids", (String[]) ids.toArray());
+
+    return query(reqParams).getResults();
+  }
+  
+  /**
+   * SolrServer implementations need to implement how a request is actually processed
+   */
+  public abstract NamedList<Object> request(final SolrRequest request) throws SolrServerException, IOException;
+
+  public DocumentObjectBinder getBinder() {
+    if(binder == null){
+      binder = new DocumentObjectBinder();
+    }
+    return binder;
+  }
+
+  /**
+   * Release allocated resources.
+   *
+   * @since solr 4.0
+   * @deprecated Use close() instead.
+   */
+  @Deprecated
+  public abstract void shutdown();
+
+  //@SuppressWarnings("deprecation")
+  public void close() throws IOException {
+    shutdown();
+  }
+}


Mime
View raw message