metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject metron git commit: METRON-1283 Install Elasticsearch template as a part of the mpack startup scripts (anandsubbu via nickwallen) closes apache/metron#817
Date Mon, 30 Oct 2017 16:23:52 GMT
Repository: metron
Updated Branches:
  refs/heads/master 191d5d84a -> 2488842bd


METRON-1283 Install Elasticsearch template as a part of the mpack startup scripts (anandsubbu
via nickwallen) closes apache/metron#817


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2488842b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2488842b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2488842b

Branch: refs/heads/master
Commit: 2488842bdfc8bf394558be93bc9bb6017759fa57
Parents: 191d5d8
Author: anandsubbu <asubramanian@hortonworks.com>
Authored: Mon Oct 30 12:23:27 2017 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Mon Oct 30 12:23:27 2017 -0400

----------------------------------------------------------------------
 .../CURRENT/package/scripts/indexing_commands.py   |  7 +++++++
 .../CURRENT/package/scripts/indexing_master.py     |  7 +++++++
 .../CURRENT/package/scripts/params/params_linux.py |  1 +
 .../package/scripts/params/status_params.py        |  3 +++
 .../roles/load_web_templates/tasks/main.yml        |  8 --------
 metron-platform/metron-elasticsearch/README.md     | 17 ++++++++++++++++-
 6 files changed, 34 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 17374eb..e6cfabb 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -48,6 +48,7 @@ class IndexingCommands:
         self.__acl_configured = os.path.isfile(self.__params.indexing_acl_configured_flag_file)
         self.__hbase_configured = os.path.isfile(self.__params.indexing_hbase_configured_flag_file)
         self.__hbase_acl_configured = os.path.isfile(self.__params.indexing_hbase_acl_configured_flag_file)
+        self.__elasticsearch_template_installed = os.path.isfile(self.__params.elasticsearch_template_installed_flag_file)
         self.__hdfs_perm_configured = os.path.isfile(self.__params.indexing_hdfs_perm_configured_flag_file)
 
     def __get_topics(self):
@@ -72,6 +73,9 @@ class IndexingCommands:
     def is_hbase_acl_configured(self):
         return self.__hbase_acl_configured
 
+    def is_elasticsearch_template_installed(self):
+        return self.__elasticsearch_template_installed
+
     def set_configured(self):
         metron_service.set_configured(self.__params.metron_user, self.__params.indexing_configured_flag_file,
"Setting Indexing configured to True")
 
@@ -87,6 +91,9 @@ class IndexingCommands:
     def set_hdfs_perm_configured(self):
         metron_service.set_configured(self.__params.metron_user, self.__params.indexing_hdfs_perm_configured_flag_file,
"Setting HDFS perm configured to True")
 
+    def set_elasticsearch_template_installed(self):
+        metron_service.set_configured(self.__params.metron_user, self.__params.elasticsearch_template_installed_flag_file,
"Setting Elasticsearch template installed to True")
+
     def create_hbase_tables(self):
         Logger.info("Creating HBase Tables for indexing")
         if self.__params.security_enabled:

http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 8f156d6..e88b949 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -80,6 +80,13 @@ class Indexing(Script):
         from params import params
         env.set_params(params)
         self.configure(env)
+        # Install elasticsearch templates
+        try:
+            if not commands.is_elasticsearch_template_installed():
+                self.elasticsearch_template_install(env)
+                commands.set_elasticsearch_template_installed()
+        except:
+            Logger.warning("WARNING: Elasticsearch templates could not be installed. The
Elasticsearch service is probably down.")
         commands = IndexingCommands(params)
         commands.start_indexing_topology(env)
 

http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index a181935..fa4c0ab 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -80,6 +80,7 @@ indexing_acl_configured_flag_file = status_params.indexing_acl_configured_flag_f
 indexing_hbase_configured_flag_file = status_params.indexing_hbase_configured_flag_file
 indexing_hbase_acl_configured_flag_file = status_params.indexing_hbase_acl_configured_flag_file
 indexing_hdfs_perm_configured_flag_file = status_params.indexing_hdfs_perm_configured_flag_file
+elasticsearch_template_installed_flag_file = status_params.elasticsearch_template_installed_flag_file
 global_properties_template = config['configurations']['metron-env']['elasticsearch-properties']
 
 # Elasticsearch hosts and port management

http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index d4f2799..a9a37d3 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -74,6 +74,9 @@ indexing_hdfs_perm_configured_flag_file = metron_zookeeper_config_path +
'/../me
 indexing_hbase_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hbase_configured'
 indexing_hbase_acl_configured_flag_file = metron_zookeeper_config_path + '/../metron_indexing_hbase_acl_configured'
 
+# Elasticsearch
+elasticsearch_template_installed_flag_file = metron_zookeeper_config_path + '/../metron_elasticsearch_template_installed_flag_file'
+
 # REST
 metron_rest_port = config['configurations']['metron-rest-env']['metron_rest_port']
 

http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-deployment/roles/load_web_templates/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/load_web_templates/tasks/main.yml b/metron-deployment/roles/load_web_templates/tasks/main.yml
index a5dbbba..3a91960 100644
--- a/metron-deployment/roles/load_web_templates/tasks/main.yml
+++ b/metron-deployment/roles/load_web_templates/tasks/main.yml
@@ -15,14 +15,6 @@
 #  limitations under the License.
 #
 ---
-- name: Load ES Templates
-  command: >
-    curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo":
{ "context": "Install ES Template from REST", "command": "ELASTICSEARCH_TEMPLATE_INSTALL"},"Requests/resource_filters":
[{"service_name": "METRON","component_name": "METRON_INDEXING","hosts" : "{{ metron_hosts[0]
}}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name
}}/requests
-  args:
-    warn: off
-  register: result
-  failed_when: "result.rc != 0 or '202' not in result.stdout"
-
 - name: Load Kibana Dashboard
   command: >
     curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo":
{ "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters":
[{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "{{ kibana_hosts[0]
}}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name
}}/requests

http://git-wip-us.apache.org/repos/asf/metron/blob/2488842b/metron-platform/metron-elasticsearch/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md
index 1e39691..9113672 100644
--- a/metron-platform/metron-elasticsearch/README.md
+++ b/metron-platform/metron-elasticsearch/README.md
@@ -2,7 +2,7 @@
 
 ## Introduction
 
-Elasticsearch can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing.README.md).
+Elasticsearch can be used as the real-time portion of the datastore resulting from [metron-indexing](../metron-indexing/README.md).
 
 ## Properties
 
@@ -81,3 +81,18 @@ curl -XPUT "http://${ELASTICSEARCH}:9200/${SENSOR}_index*/_mapping/${SENSOR}_doc
 '
 rm ${SENSOR}.template
 ```
+
+## Installing Elasticsearch Templates
+
+The stock set of Elasticsearch templates for bro, snort, yaf, error index and meta index
are installed automatically during the first time install and startup of Metron Indexing service.
+
+It is possible that Elasticsearch service is not available when the Metron Indexing Service
startup, in that case the Elasticsearch template will not be installed. 
+
+For such a scenario, an Admin can have the template installed in two ways:
+
+_Method 1_ - Manually from the Ambari UI by following the flow:
+Ambari UI -> Services -> Metron -> Service Actions -> Elasticsearch Template
Install
+
+_Method 2_ - Stop the Metron Indexing service, and start it again from Ambari UI. Note that
the Metron Indexing service tracks if it has successfully installed the Elasticsearch templates,
and will attempt to do so each time it is Started until successful.
+
+> Note: If you have made any customization to your index templates, then installing Elasticsearch
templates afresh will lead to overwriting your existing changes. Please exercise caution.


Mime
View raw message