metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mmiklav...@apache.org
Subject [19/50] [abbrv] metron git commit: METRON-1587 Make collection utility work for HDP search (merrimanr) closes apache/metron#1043
Date Wed, 11 Jul 2018 01:32:35 GMT
METRON-1587 Make collection utility work for HDP search (merrimanr) closes apache/metron#1043


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f241f87f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f241f87f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f241f87f

Branch: refs/heads/feature/METRON-1554-pcap-query-panel
Commit: f241f87f87523f2c6f2203b3f316cff0fd3f20b4
Parents: aed4ffc
Author: merrimanr <merrimanr@gmail.com>
Authored: Mon Jun 11 10:09:16 2018 -0500
Committer: merrimanr <merrimanr@gmail.com>
Committed: Mon Jun 11 10:09:16 2018 -0500

----------------------------------------------------------------------
 .../package/scripts/indexing_commands.py        | 63 +++++++++++++++++---
 .../CURRENT/package/scripts/indexing_master.py  | 31 +---------
 .../package/scripts/params/params_linux.py      |  5 ++
 metron-platform/metron-solr/README.md           | 33 +++++++++-
 .../src/main/scripts/create_collection.sh       | 21 +++++--
 .../src/main/scripts/delete_collection.sh       | 18 ++++--
 6 files changed, 120 insertions(+), 51 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index eeb2127..69e980b 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -25,6 +25,7 @@ from resource_management.core.exceptions import Fail
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute, File
 from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.functions.format import format
 
 import metron_service
 import metron_security
@@ -85,14 +86,13 @@ class IndexingCommands:
         :return: Dict where key is the name of a collection and the
           value is a path to file containing the schema definition.
         """
-        from params import params
-        return {
-            "bro": params.bro_schema_path,
-            "yaf": params.yaf_schema_path,
-            "snort": params.snort_schema_path,
-            "error": params.error_schema_path,
-            "metaalert": params.meta_schema_path
-        }
+        return [
+            "bro",
+            "yaf",
+            "snort",
+            "error",
+            "metaalert"
+        ]
 
     def is_configured(self):
         return self.__configured
@@ -200,6 +200,53 @@ class IndexingCommands:
               user=self.__params.metron_user,
               err_msg=err_msg.format(template_name))
 
+    def solr_schema_install(self, env):
+        from params import params
+        env.set_params(params)
+        Logger.info("Installing Solr schemas")
+        if self.__params.security_enabled:
+            metron_security.kinit(self.__params.kinit_path_local,
+                                  self.__params.solr_keytab_path,
+                                  self.__params.solr_principal_name,
+                                  self.__params.solr_user)
+
+        commands = IndexingCommands(params)
+        for collection_name in commands.get_solr_schemas():
+
+            # install the schema
+            cmd = format((
+                "export ZOOKEEPER={solr_zookeeper_url};"
+                "export SECURITY_ENABLED={security_enabled};"
+            ))
+            cmd += "{0}/bin/create_collection.sh {1};"
+
+            Execute(
+                cmd.format(params.metron_home, collection_name),
+                user=self.__params.solr_user)
+
+    def solr_schema_delete(self, env):
+        from params import params
+        env.set_params(params)
+        Logger.info("Deleting Solr schemas")
+        if self.__params.security_enabled:
+            metron_security.kinit(self.__params.kinit_path_local,
+                                  self.__params.solr_keytab_path,
+                                  self.__params.solr_principal_name,
+                                  self.__params.solr_user)
+
+        commands = IndexingCommands(params)
+        for collection_name in commands.get_solr_schemas():
+            # delete the schema
+            cmd = format((
+                "export ZOOKEEPER={solr_zookeeper_url};"
+                "export SECURITY_ENABLED={security_enabled};"
+            ))
+            cmd += "{0}/bin/delete_collection.sh {1};"
+
+            Execute(
+                cmd.format(params.metron_home, collection_name),
+                user=self.__params.solr_user)
+
     def start_batch_indexing_topology(self, env):
         Logger.info('Starting ' + self.__batch_indexing_topology)
 

http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index b5c4bb9..1629465 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -97,11 +97,10 @@ class Indexing(Script):
         self.configure(env)
         commands = IndexingCommands(params)
         if params.ra_indexing_writer == 'Solr':
-            Logger.info("Loading Solr schemas")
             # Install Solr schemas
             try:
                 if not commands.is_solr_schema_installed():
-                    self.solr_schema_install(env)
+                    commands.solr_schema_install(env)
                     commands.set_solr_schema_installed()
 
             except Exception as e:
@@ -170,34 +169,6 @@ class Indexing(Script):
               cmd.format(params.es_http_url, template_name),
               logoutput=True)
 
-    def solr_schema_install(self, env):
-        from params import params
-        env.set_params(params)
-        Logger.info("Installing Solr schemas")
-
-        commands = IndexingCommands(params)
-        for collection_name, config_path in commands.get_solr_schemas().iteritems():
-
-            # install the schema
-
-            cmd = "{0}/bin/solr create -c {1} -d {2}"
-            Execute(
-                cmd.format(params.solr_home, collection_name, config_path),
-                logoutput=True, user="solr")
-
-    def solr_schema_delete(self, env):
-        from params import params
-        env.set_params(params)
-        Logger.info("Deleting Solr schemas")
-
-        commands = IndexingCommands(params)
-        for collection_name, config_path in commands.get_solr_schemas().iteritems():
-            # delete the schema
-            cmd = "{0}/bin/solr delete -c {1}"
-            Execute(
-                cmd.format(params.solr_home, collection_name),
-                logoutput=True, user="solr")
-
     @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
     def kibana_dashboard_install(self, env):
       from params import params

http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 6f4760b..667a926 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -128,6 +128,9 @@ if has_zk_host:
 solr_version = '6.6.2'
 solr_home = '/var/solr/solr-' + solr_version
 solr_zookeeper_url = format(format(config['configurations']['metron-env']['solr_zookeeper_url']))
+solr_user = config['configurations']['solr-config-env']['solr_config_user']
+solr_principal_name = config['configurations']['solr-config-env']['solr_principal_name']
+solr_keytab_path = config['configurations']['solr-config-env']['solr_keytab_path']
 
 # Storm
 storm_rest_addr = status_params.storm_rest_addr
@@ -256,6 +259,8 @@ if security_enabled:
 
     nimbus_seeds = config['configurations']['storm-site']['nimbus.seeds']
 
+    solr_principal_name = solr_principal_name.replace('_HOST', hostname_lowercase)
+
 # Management UI
 metron_rest_host = default("/clusterHostInfo/metron_rest_hosts", [hostname])[0]
 

http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/README.md b/metron-platform/metron-solr/README.md
index 0d525ec..159779c 100644
--- a/metron-platform/metron-solr/README.md
+++ b/metron-platform/metron-solr/README.md
@@ -20,7 +20,10 @@ limitations under the License.
 ## Table of Contents
 
 * [Introduction](#introduction)
+* [Configuration](#configuration)
 * [Installing](#installing)
+* [Schemas](#schemas)
+* [Collections](#collections)
 
 ## Introduction
 
@@ -110,4 +113,32 @@ A PointType field should be defined as:
 <dynamicField name="*_point" type="pdouble" indexed="true" stored="false" docValues="false"/>
 <fieldType name="point" class="solr.PointType" subFieldSuffix="_point"/>
 ```
-If any copy fields are defined, stored and docValues should be set to false.
\ No newline at end of file
+If any copy fields are defined, stored and docValues should be set to false.
+
+## Collections
+
+Convenience scripts are provided with Metron to create and delete collections.  Ambari uses
these scripts to automatically create collections.  To use them outside of Ambari, a few environment
variables must be set first:
+```
+# Path to the zookeeper node used by Solr
+export ZOOKEEPER=node1:2181/solr
+# Set to true if Kerberos is enabled
+export SECURITY_ENABLED=true 
+```
+The scripts can then be called directly with the collection name as the first argument .
 For example, to create the bro collection:
+```
+$METRON_HOME/bin/create_collection.sh bro
+```
+To delete the bro collection:
+```
+$METRON_HOME/bin/delete_collection.sh bro
+```
+The `create_collection.sh` script depends on schemas installed in `$METRON_HOME/config/schema`.
 There are several schemas that come with Metron:
+
+* bro
+* snort
+* yaf
+* metaalert
+* error
+
+Additional schemas should be installed in that location if using the `create_collection.sh`
script.  Any collection can be deleted with the `delete_collection.sh` script.
+These scripts use the [Solr Collection API](http://lucene.apache.org/solr/guide/6_6/collections-api.html).
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/src/main/scripts/create_collection.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/create_collection.sh b/metron-platform/metron-solr/src/main/scripts/create_collection.sh
index 37fbccb..7693646 100755
--- a/metron-platform/metron-solr/src/main/scripts/create_collection.sh
+++ b/metron-platform/metron-solr/src/main/scripts/create_collection.sh
@@ -18,10 +18,19 @@
 #
 METRON_VERSION=${project.version}
 METRON_HOME=/usr/metron/$METRON_VERSION
-SOLR_VERSION=${global_solr_version}
-SOLR_USER=solr
-SOLR_SERVICE=$SOLR_USER
-SOLR_VAR_DIR="/var/$SOLR_SERVICE"
+ZOOKEEPER=${ZOOKEEPER:-localhost:2181}
+ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/hdp/current/zookeeper-client}
+SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+NEGOTIATE=''
+if [ ${SECURITY_ENABLED,,} == 'true' ]; then
+    NEGOTIATE=' --negotiate -u : '
+fi
 
-cd $SOLR_VAR_DIR/solr-${SOLR_VERSION}
-su $SOLR_USER -c "bin/solr create -c $1 -d $METRON_HOME/config/schema/$1/"
+# Get the first Solr node from the list of live nodes in Zookeeper
+SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail -n 1 | sed
's/\[\([^,]*\).*\]/\1/' | sed 's/_solr//'`
+
+# Upload the collection config set
+zip -rj - $METRON_HOME/config/schema/$1 | curl -X POST $NEGOTIATE --header "Content-Type:text/xml"
--data-binary @- "http://$SOLR_NODE/solr/admin/configs?action=UPLOAD&name=$1"
+
+# Create the collection
+curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=CREATE&name=$1&numShards=1"

http://git-wip-us.apache.org/repos/asf/metron/blob/f241f87f/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
index 68f0c20..c8b45e7 100755
--- a/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
+++ b/metron-platform/metron-solr/src/main/scripts/delete_collection.sh
@@ -18,10 +18,16 @@
 #
 METRON_VERSION=${project.version}
 METRON_HOME=/usr/metron/$METRON_VERSION
-SOLR_VERSION=${global_solr_version}
-SOLR_USER=solr
-SOLR_SERVICE=$SOLR_USER
-SOLR_VAR_DIR="/var/$SOLR_SERVICE"
+ZOOKEEPER=${ZOOKEEPER:-localhost:2181}
+ZOOKEEPER_HOME=${ZOOKEEPER_HOME:-/usr/hdp/current/zookeeper-client}
+SECURITY_ENABLED=${SECURITY_ENABLED:-false}
+NEGOTIATE=''
+if [ ${SECURITY_ENABLED,,} == 'true' ]; then
+    NEGOTIATE=' --negotiate -u : '
+fi
 
-cd $SOLR_VAR_DIR/solr-${SOLR_VERSION}
-su $SOLR_USER -c "bin/solr delete -c $1"
+# Get the first Solr node from the list of live nodes in Zookeeper
+SOLR_NODE=`$ZOOKEEPER_HOME/bin/zkCli.sh -server $ZOOKEEPER ls /live_nodes | tail -n 1 | sed
's/\[\([^,]*\).*\]/\1/' | sed 's/_solr//'`
+
+# Delete the collection
+curl -X GET $NEGOTIATE "http://$SOLR_NODE/solr/admin/collections?action=DELETE&name=$1"


Mime
View raw message