ranger-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject [3/3] incubator-ranger git commit: RANGER-728: Update Solr script to resolve issues with ZK and creating collection
Date Thu, 10 Dec 2015 20:17:53 GMT
RANGER-728: Update Solr script to resolve issues with ZK and creating
collection


Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/269617d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/269617d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/269617d5

Branch: refs/heads/master
Commit: 269617d5dbf13fcbf9600efca72bc5a803f49a92
Parents: bb81459
Author: Don Bosco Durai <bosco@apache.org>
Authored: Wed Dec 9 16:23:11 2015 -0800
Committer: Don Bosco Durai <bosco@apache.org>
Committed: Thu Dec 10 12:17:19 2015 -0800

----------------------------------------------------------------------
 .../create_hdfs_folders_for_audit_secure.sh     |    4 +-
 .../solr_for_audit_setup/conf/managed-schema    |   92 +
 .../solr_for_audit_setup/conf/schema.xml        |  118 --
 .../solr_for_audit_setup/conf/solrconfig.xml    |   13 +
 .../solr_for_audit_setup/conf/solrconfig.xml.j2 | 1878 ++++++++++++++++++
 .../solr_for_audit_setup/install.properties     |    3 +
 .../resources/log4j.properties.j2               |   40 +
 .../resources/log4j.properties.template         |   39 -
 .../contrib/solr_for_audit_setup/setup.sh       |   31 +-
 .../scripts/add_ranger_audits_conf_to_zk.sh.j2  |   63 +
 .../add_ranger_audits_conf_to_zk.sh.template    |   63 -
 .../create_ranger_audits_collection.sh.j2       |   33 +
 .../create_ranger_audits_collection.sh.template |   33 -
 .../solr_cloud/scripts/solr.in.sh.j2            |  116 ++
 .../solr_cloud/scripts/solr.sh.j2               |   21 +
 .../solr_cloud/scripts/start_solr.sh.j2         |   32 +
 .../solr_cloud/scripts/start_solr.sh.template   |   39 -
 .../solr_cloud/scripts/stop_solr.sh.j2          |   33 +
 .../solr_cloud/scripts/stop_solr.sh.template    |   35 -
 .../solr_for_audit_setup/solr_cloud/solr.xml.j2 |   26 +
 .../solr_cloud/solr.xml.template                |   26 -
 .../ranger_audits/core.properties.j2            |   20 +
 .../ranger_audits/core.properties.template      |   20 -
 .../solr_standalone/scripts/solr.in.sh.j2       |  116 ++
 .../solr_standalone/scripts/solr.sh.j2          |   21 +
 .../solr_standalone/scripts/start_solr.sh.j2    |   33 +
 .../scripts/start_solr.sh.template              |   38 -
 .../solr_standalone/scripts/stop_solr.sh.j2     |   33 +
 .../scripts/stop_solr.sh.template               |   35 -
 29 files changed, 2593 insertions(+), 461 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/269617d5/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
----------------------------------------------------------------------
diff --git a/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
b/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
index 12a4c93..9a5bdd5 100755
--- a/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
+++ b/security-admin/contrib/audit_hdfs_folders/create_hdfs_folders_for_audit_secure.sh
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#Usage: Run this script as user hdfs or the HDFS admin user. 
+#Usage: Use this script in kerberos enabled hadoop only. Run this script after kinit'ing
as hdfs user
 #This script creates the folders in HDFS required by Apache Ranger for writing Audit records
-#Note 1: Use this script only for non-kerberos environment. In non-kerberos environment,
Ranger KMS writes the audit logs as user "HTTP"
+#Note 1: Use this script only for kerberos environment. In kerberos environment, Ranger KMS
writes the audit logs as user "HTTP"
 #Note 2: Please update the below variables according to your environment
 
 HBASE_USER_GROUP=hbase:hbase

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/269617d5/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
----------------------------------------------------------------------
diff --git a/security-admin/contrib/solr_for_audit_setup/conf/managed-schema b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
new file mode 100644
index 0000000..7b9769a
--- /dev/null
+++ b/security-admin/contrib/solr_for_audit_setup/conf/managed-schema
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+-->
+<schema name="ranger-audit-schema" version="1.5">
+  <uniqueKey>id</uniqueKey>
+  <fieldType name="binary" class="solr.BinaryField"/>
+  <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
+  <fieldType name="booleans" class="solr.BoolField" multiValued="true" sortMissingLast="true"/>
+  <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
+  <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
+  <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
+  <fieldType name="ignored" class="solr.StrField" multiValued="true" indexed="false" stored="false"/>
+  <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
+  <fieldType name="key_lower_case" class="solr.TextField" sortMissingLast="true" omitNorms="true">
+    <analyzer>
+      <tokenizer class="solr.KeywordTokenizerFactory"/>
+      <filter class="solr.LowerCaseFilterFactory"/>
+    </analyzer>
+  </fieldType>
+  <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
+  <fieldType name="random" class="solr.RandomSortField" indexed="true"/>
+  <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+  <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
+  <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" multiValued="true"
positionIncrementGap="0"/>
+  <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
+  <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" multiValued="true"
positionIncrementGap="0"/>
+  <fieldType name="text_std_token_lower_case" class="solr.TextField" multiValued="true"
positionIncrementGap="100">
+    <analyzer>
+      <tokenizer class="solr.StandardTokenizerFactory"/>
+      <filter class="solr.LowerCaseFilterFactory"/>
+    </analyzer>
+  </fieldType>
+  <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
+    <analyzer>
+      <tokenizer class="solr.WhitespaceTokenizerFactory"/>
+    </analyzer>
+  </fieldType>
+  <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
+  <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" multiValued="true"
positionIncrementGap="0"/>
+  <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
+  <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" multiValued="true"
positionIncrementGap="0"/>
+  <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
+  <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" multiValued="true"
positionIncrementGap="0"/>
+  <field name="_expire_at_" type="tdate" multiValued="false" stored="true"/>
+  <field name="_ttl_" type="string" multiValued="false" indexed="true" stored="true"/>
+  <field name="_version_" type="long" indexed="true" stored="true"/>
+  <field name="access" type="key_lower_case" multiValued="false"/>
+  <field name="action" type="key_lower_case" multiValued="false"/>
+  <field name="agent" type="key_lower_case" multiValued="false"/>
+  <field name="agentHost" type="key_lower_case" multiValued="false"/>
+  <field name="cliIP" type="key_lower_case" multiValued="false"/>
+  <field name="cliType" type="key_lower_case" multiValued="false"/>
+  <field name="cluster" type="key_lower_case" multiValued="false"/>
+  <field name="reqContext" type="key_lower_case" multiValued="true"/>
+  <field name="enforcer" type="key_lower_case" multiValued="false"/>
+  <field name="event_count" type="tlong" multiValued="false" docValues="true"/>
+  <field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/>
+  <field name="evtTime" type="tdate"/>
+  <field name="id" type="string" multiValued="false" indexed="true" required="true" stored="true"/>
+  <field name="logType" type="key_lower_case" multiValued="false"/>
+  <field name="policy" type="tlong" docValues="true"/>
+  <field name="proxyUsers" type="key_lower_case" multiValued="true"/>
+  <field name="reason" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/>
+  <field name="repo" type="key_lower_case" multiValued="false"/>
+  <field name="repoType" type="tint" multiValued="false" docValues="true"/>
+  <field name="req_caller_id" type="key_lower_case" multiValued="false"/>
+  <field name="req_self_id" type="key_lower_case" multiValued="false"/>
+  <field name="reqData" type="text_std_token_lower_case" multiValued="false"/>
+  <field name="reqUser" type="key_lower_case" multiValued="false"/>
+  <field name="resType" type="key_lower_case" multiValued="false"/>
+  <field name="resource" type="key_lower_case" multiValued="false"/>
+  <field name="result" type="tint" multiValued="false"/>
+  <field name="seq_num" type="tlong" multiValued="false" docValues="true"/>
+  <field name="sess" type="key_lower_case" multiValued="false"/>
+  <field name="tags" type="key_lower_case" multiValued="true"/>
+  <field name="tags_str" type="text_std_token_lower_case" multiValued="false"/>
+  <field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true"
stored="false"/>
+</schema>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/269617d5/security-admin/contrib/solr_for_audit_setup/conf/schema.xml
----------------------------------------------------------------------
diff --git a/security-admin/contrib/solr_for_audit_setup/conf/schema.xml b/security-admin/contrib/solr_for_audit_setup/conf/schema.xml
deleted file mode 100644
index df56974..0000000
--- a/security-admin/contrib/solr_for_audit_setup/conf/schema.xml
+++ /dev/null
@@ -1,118 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements.  See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License.  You may obtain a copy of the License at
-
-     http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Trimmed schema.xml to include only the fields used by RangerAudit and also to make
it more manageable -->
-<schema name="ranger-audit-schema" version="1.5">
-  <fields>
-    <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false"
/>
-    <field name="_version_" type="long" indexed="true" stored="true"/>
-
-    <field name="resource" type="key_lower_case" multiValued="false"/>
-    <field name="cluster" type="key_lower_case" multiValued="false"/>
-    <field name="resType" type="key_lower_case" multiValued="false"/>
-    <field name="reqUser" type="key_lower_case" multiValued="false"/>
-    <field name="evtTime" type="tdate"/>
-    <field name="policy" type="tlong" docValues="true"/>
-    <field name="sess" type="key_lower_case" multiValued="false"/>
-    <field name="access" type="key_lower_case" multiValued="false"/>
-    <field name="result" type="tint" multiValued="false"/>
-    <field name="reason" type="text_std_token_lower_case" omitNorms="false" multiValued="false"/>
-    <field name="enforcer" type="key_lower_case" multiValued="false"/>
-    <field name="repo" type="key_lower_case" multiValued="false"/>
-    <field name="cliIP" type="key_lower_case" multiValued="false"/>
-    <field name="cliType" type="key_lower_case" multiValued="false"/>
-    <field name="action" type="key_lower_case" multiValued="false"/>
-    <field name="agent" type="key_lower_case" multiValued="false"/>
-    <field name="agentHost" type="key_lower_case" multiValued="false"/>
-    <field name="logType" type="key_lower_case" multiValued="false"/>
-    <field name="repoType" type="tint" multiValued="false" docValues="true"/>
-    <field name="reqData" type="text_std_token_lower_case" multiValued="false"/>
-    <field name="seq_num" type="tlong" multiValued="false" docValues="true"/>
-    <field name="tags" type="key_lower_case" multiValued="true"/>
-    <field name="tags_str" type="text_std_token_lower_case" multiValued="false"/>
-    <field name="event_count" type="tlong" multiValued="false" docValues="true"/>
-    <field name="event_dur_ms" type="tlong" multiValued="false" docValues="true"/>
-    
-    <field name="text" type="text_std_token_lower_case" indexed="true" stored="false"
multiValued="true"/>
-  </fields>
-
-  <uniqueKey>id</uniqueKey>
-
-   <types>
-
-    <!-- Updated from text_general and having only StandardTokenizer and LowerCaseFilter-->
-    <fieldType name="text_std_token_lower_case" class="solr.TextField" positionIncrementGap="100"
multiValued="true">
-      <analyzer>
-	<tokenizer class="solr.StandardTokenizerFactory"/>
-        <filter class="solr.LowerCaseFilterFactory" />
-      </analyzer>
-    </fieldType>
-
-    <!-- Keyword tokenizer won't split the string. So is like full string search -->
-    <fieldType name="key_lower_case" class="solr.TextField"
-               sortMissingLast="true" omitNorms="true">
-      <analyzer>
-        <tokenizer class="solr.KeywordTokenizerFactory"/>
-        <filter class="solr.LowerCaseFilterFactory" />
-      </analyzer>
-    </fieldType>
-
-    <!-- The StrField type is not analyzed, but indexed/stored verbatim.
-       It supports doc values but in that case the field needs to be
-       single-valued and either required or have a default value.
-      -->
-    <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
-
-    <!-- boolean type: "true" or "false" -->
-    <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
-
-    <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/>
-
-    <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
-    <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
-    <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
-    <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
-
-    <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
-    <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
-    <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
-    <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
-    
-    <fieldType name="tints" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"
multiValued="true"/>
-    <fieldType name="tfloats" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"
multiValued="true"/>
-    <fieldType name="tlongs" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"
multiValued="true"/>
-    <fieldType name="tdoubles" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"
multiValued="true"/>
-    <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
-    <!-- A Trie based date field for faster date range queries and date faceting. -->
-    <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
-    <fieldType name="tdates" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"
multiValued="true"/>
-    <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings
-->
-    <fieldtype name="binary" class="solr.BinaryField"/>
-    <fieldType name="random" class="solr.RandomSortField" indexed="true" />
-    <!-- A text field that only splits on whitespace for exact matching of words -->
-    <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
-      <analyzer>
-        <tokenizer class="solr.WhitespaceTokenizerFactory"/>
-      </analyzer>
-    </fieldType>
-    <!-- since fields of this type are by default not stored or indexed,
-         any data added to them will be ignored outright.  --> 
-    <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField"
/>
-
-  </types>
-</schema>

http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/269617d5/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml
----------------------------------------------------------------------
diff --git a/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml b/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml
index 0991eba..7e71f9a 100644
--- a/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml
+++ b/security-admin/contrib/solr_for_audit_setup/conf/solrconfig.xml
@@ -1642,6 +1642,19 @@
        See http://wiki.apache.org/solr/GuessingFieldTypes
     -->
   <updateRequestProcessorChain name="add-unknown-fields-to-the-schema">
+    <processor class="solr.DefaultValueUpdateProcessorFactory">
+        <str name="fieldName">_ttl_</str>
+        <str name="value">+1095DAYS</str>
+    </processor>
+    <processor class="solr.processor.DocExpirationUpdateProcessorFactory">
+        <int name="autoDeletePeriodSeconds">300</int>
+        <str name="ttlFieldName">_ttl_</str>
+        <str name="expirationFieldName">_expire_at_</str>
+    </processor>
+    <processor class="solr.FirstFieldValueUpdateProcessorFactory">
+      <str name="fieldName">_expire_at_</str>
+    </processor>
+
     <processor class="solr.RemoveBlankFieldUpdateProcessorFactory"/>
     <processor class="solr.ParseBooleanFieldUpdateProcessorFactory"/>
     <processor class="solr.ParseLongFieldUpdateProcessorFactory"/>


Mime
View raw message