metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nickal...@apache.org
Subject metron git commit: METRON-1609 Elasticsearch settings in Ambari should not be required if Solr is the indexer (nickwallen) closes apache/metron#1056
Date Thu, 21 Jun 2018 15:30:05 GMT
Repository: metron
Updated Branches:
  refs/heads/feature/METRON-1416-upgrade-solr 2bf665032 -> ed4dfb936


METRON-1609 Elasticsearch settings in Ambari should not be required if Solr is the indexer
(nickwallen) closes apache/metron#1056


Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ed4dfb93
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ed4dfb93
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ed4dfb93

Branch: refs/heads/feature/METRON-1416-upgrade-solr
Commit: ed4dfb93641c9c9a84b61b5daddbebd962b3b20c
Parents: 2bf6650
Author: nickwallen <nick@nickallen.org>
Authored: Thu Jun 21 11:28:30 2018 -0400
Committer: nickallen <nickallen@apache.org>
Committed: Thu Jun 21 11:28:30 2018 -0400

----------------------------------------------------------------------
 .../METRON/CURRENT/configuration/metron-env.xml |  32 ++++--
 .../package/scripts/indexing_commands.py        |   4 +-
 .../CURRENT/package/scripts/indexing_master.py  |  10 +-
 .../CURRENT/package/scripts/metron_service.py   | 101 ++++++++++++++-----
 4 files changed, 113 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
index 0b64f3a..cdef7cf 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-env.xml
@@ -81,30 +81,54 @@
         <value>metron</value>
         <description>Name of Elasticsearch Cluster</description>
         <display-name>Elasticsearch Cluster Name</display-name>
+        <value-attributes>
+            <empty-value-valid>true</empty-value-valid>
+        </value-attributes>
     </property>
-    <property require-input="true">
+    <property>
         <name>es_hosts</name>
         <value></value>
         <description>Comma delimited list of Elasticsearch Master Hosts: eshost1,eshost2</description>
         <display-name>Elasticsearch Hosts</display-name>
+        <value-attributes>
+            <empty-value-valid>true</empty-value-valid>
+        </value-attributes>
     </property>
     <property>
         <name>es_binary_port</name>
         <value>9300</value>
         <description>Elasticsearch binary port. (9300)</description>
         <display-name>Elasticsearch Binary Port</display-name>
+        <value-attributes>
+            <empty-value-valid>true</empty-value-valid>
+        </value-attributes>
     </property>
     <property>
         <name>es_http_port</name>
         <value>9200</value>
         <description>Elasticsearch HTTP port. (9200)</description>
         <display-name>Elasticsearch HTTP port</display-name>
+        <value-attributes>
+            <empty-value-valid>true</empty-value-valid>
+        </value-attributes>
+    </property>
+    <property>
+        <name>es_date_format</name>
+        <description>Elasticsearch Date Format</description>
+        <value>yyyy.MM.dd.HH</value>
+        <display-name>Elasticsearch Date Format</display-name>
+        <value-attributes>
+            <empty-value-valid>true</empty-value-valid>
+        </value-attributes>
     </property>
     <property>
         <name>solr_zookeeper_url</name>
         <value>{{zookeeper_quorum}}</value>
         <description>Comma delimited list of Zookeeper Urls: zkhost1:2181,zkhost1:2181</description>
         <display-name>Solr Zookeeper Urls</display-name>
+        <value-attributes>
+            <empty-value-valid>true</empty-value-valid>
+        </value-attributes>
     </property>
     <property require-input = "true">
         <name>storm_rest_addr</name>
@@ -137,10 +161,4 @@
             <empty-value-valid>true</empty-value-valid>
         </value-attributes>
     </property>
-    <property>
-        <name>es_date_format</name>
-        <description>Elasticsearch Date Format</description>
-        <value>yyyy.MM.dd.HH</value>
-        <display-name>Elasticsearch Date Format</display-name>
-    </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 69e980b..4802add 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -30,7 +30,6 @@ from resource_management.libraries.functions.format import format
 import metron_service
 import metron_security
 
-
 # Wrap major operations and functionality in this class
 class IndexingCommands:
     __params = None
@@ -394,7 +393,6 @@ class IndexingCommands:
             is_random_access_running = topologies[self.__random_access_indexing_topology]
in ['ACTIVE', 'REBALANCING']
         return is_random_access_running
 
-
     def is_topology_active(self, env):
         return self.is_batch_topology_active(env) and self.is_random_access_topology_active(env)
 
@@ -403,6 +401,8 @@ class IndexingCommands:
         Performs a service check for Indexing.
         :param env: Environment
         """
+        metron_service.check_indexer_parameters()
+
         Logger.info('Checking Kafka topics for Indexing')
         metron_service.check_kafka_topics(self.__params, self.__get_topics())
 

http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 1629465..9f9ab87 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -48,6 +48,7 @@ class Indexing(Script):
         env.set_params(params)
 
         Logger.info("Running indexing configure")
+        metron_service.check_indexer_parameters()
         File(format("{metron_config_path}/elasticsearch.properties"),
              content=Template("elasticsearch.properties.j2"),
              owner=params.metron_user,
@@ -137,6 +138,7 @@ class Indexing(Script):
     def restart(self, env):
         from params import params
         env.set_params(params)
+
         self.configure(env)
         commands = IndexingCommands(params)
         commands.restart_indexing_topology(env)
@@ -145,6 +147,7 @@ class Indexing(Script):
         from params import params
         env.set_params(params)
         Logger.info("Installing Elasticsearch index templates")
+        metron_service.check_indexer_parameters()
 
         commands = IndexingCommands(params)
         for template_name, template_path in commands.get_templates().iteritems():
@@ -160,9 +163,11 @@ class Indexing(Script):
         from params import params
         env.set_params(params)
         Logger.info("Deleting Elasticsearch index templates")
+        metron_service.check_indexer_parameters()
 
         commands = IndexingCommands(params)
         for template_name in commands.get_templates():
+
             # delete the index template
             cmd = "curl -s -XDELETE \"http://{0}/_template/{1}\""
             Execute(
@@ -173,9 +178,9 @@ class Indexing(Script):
     def kibana_dashboard_install(self, env):
       from params import params
       env.set_params(params)
+      metron_service.check_indexer_parameters()
 
       Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url))
-
       kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard',
'kibana.template')
       if not os.path.isfile(kibanaTemplate):
         raise IOError(
@@ -200,8 +205,9 @@ class Indexing(Script):
     def zeppelin_notebook_import(self, env):
         from params import params
         env.set_params(params)
-        commands = IndexingCommands(params)
+        metron_service.check_indexer_parameters()
 
+        commands = IndexingCommands(params)
         Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))
 
         # Check if authentication is configured on Zeppelin server, and fetch details if
enabled.

http://git-wip-us.apache.org/repos/asf/metron/blob/ed4dfb93/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index 23482e3..4a55f8a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -27,6 +27,7 @@ from resource_management.core.resources.system import Execute
 from resource_management.core.source import InlineTemplate
 from resource_management.libraries.functions import format as ambari_format
 from resource_management.libraries.functions.get_user_call_output import get_user_call_output
+from resource_management.libraries.script import Script
 
 from metron_security import kinit
 
@@ -51,10 +52,23 @@ def set_configured(user, flag_file, log_msg):
 def set_zk_configured(params):
   set_configured(params.metron_user, params.zk_configured_flag_file, "Setting Zookeeper configured
to true")
 
-def build_global_config_patch(params, patch_file):
-  # see RFC 6902 at https://tools.ietf.org/html/rfc6902
-  patch_template = """
-  [
+def solr_global_config_patches():
+  """
+  Builds the global configuration patches required for Solr.
+  """
+  return """
+    {
+        "op": "add",
+        "path": "/solr.zookeeper",
+        "value": "{{solr_zookeeper_url}}"
+    }
+  """
+
+def elasticsearch_global_config_patches():
+  """
+  Builds the global configuration patches required for Elasticsearch.
+  """
+  return """
     {
         "op": "add",
         "path": "/es.clustername",
@@ -69,11 +83,31 @@ def build_global_config_patch(params, patch_file):
         "op": "add",
         "path": "/es.date.format",
         "value": "{{es_date_format}}"
+    }
+  """
+
+def build_global_config_patch(params, patch_file):
+  """
+  Build the file used to patch the global configuration.
+  See RFC 6902 at https://tools.ietf.org/html/rfc6902
+
+  :param params:
+  :param patch_file: The path where the patch file will be created.
+  """
+  if params.ra_indexing_writer == 'Solr':
+      indexing_patches = solr_global_config_patches()
+  else:
+      indexing_patches = elasticsearch_global_config_patches()
+  other_patches = """
+    {
+        "op": "add",
+        "path": "/profiler.client.period.duration",
+        "value": "{{profiler_period_duration}}"
     },
     {
         "op": "add",
-        "path": "/solr.zookeeper",
-        "value": "{{solr_zookeeper_url}}"
+        "path": "/profiler.client.period.duration.units",
+        "value": "{{profiler_period_units}}"
     },
     {
         "op": "add",
@@ -92,16 +126,6 @@ def build_global_config_patch(params, patch_file):
     },
     {
         "op": "add",
-        "path": "/profiler.client.period.duration",
-        "value": "{{profiler_period_duration}}"
-    },
-    {
-        "op": "add",
-        "path": "/profiler.client.period.duration.units",
-        "value": "{{profiler_period_units}}"
-    },
-    {
-        "op": "add",
         "path": "/user.settings.hbase.table",
         "value": "{{user_settings_hbase_table}}"
     },
@@ -115,8 +139,14 @@ def build_global_config_patch(params, patch_file):
         "path": "/bootstrap.servers",
         "value": "{{kafka_brokers}}"
     }
-  ]
   """
+  patch_template = ambari_format(
+  """
+  [
+    {indexing_patches},
+    {other_patches}
+  ]
+  """)
   File(patch_file,
        content=InlineTemplate(patch_template),
        owner=params.metron_user,
@@ -132,6 +162,7 @@ def patch_global_config(params):
       "{metron_home}/bin/zk_load_configs.sh --zk_quorum {zookeeper_quorum} --mode PATCH --config_type
GLOBAL --patch_file " + patch_file),
       path=ambari_format("{java_home}/bin")
   )
+  Logger.info("Done patching global config")
 
 def pull_config(params):
   Logger.info('Pulling all Metron configs down from ZooKeeper to local file system')
@@ -141,17 +172,12 @@ def pull_config(params):
       path=ambari_format("{java_home}/bin")
   )
 
-# pushes json patches to zookeeper based on Ambari parameters that are configurable by the
user
 def refresh_configs(params):
   if not is_zk_configured(params):
     Logger.warning("The expected flag file '" + params.zk_configured_flag_file + "'indicating
that Zookeeper has been configured does not exist. Skipping patching. An administrator should
look into this.")
     return
-
-  Logger.info("Patch global config in Zookeeper")
+  check_indexer_parameters()
   patch_global_config(params)
-  Logger.info("Done patching global config")
-
-  Logger.info("Pull zookeeper config locally")
   pull_config(params)
 
 def get_running_topologies(params):
@@ -505,3 +531,32 @@ def check_http(host, port, user):
       Execute(cmd, tries=3, try_sleep=5, logoutput=False, user=user)
     except:
       raise ComponentIsNotRunning()
+
+def check_indexer_parameters():
+    """
+    Ensure that all required parameters have been defined for the chosen
+    Indexer; either Solr or Elasticsearch.
+    """
+    missing = []
+    config = Script.get_config()
+    indexer = config['configurations']['metron-indexing-env']['ra_indexing_writer']
+    Logger.info('Checking parameters for indexer = ' + indexer)
+
+    if indexer == 'Solr':
+      # check for all required solr parameters
+      if not config['configurations']['metron-env']['solr_zookeeper_url']:
+        missing.append("metron-env/solr_zookeeper_url")
+
+    else:
+      # check for all required elasticsearch parameters
+      if not config['configurations']['metron-env']['es_cluster_name']:
+        missing.append("metron-env/es_cluster_name")
+      if not config['configurations']['metron-env']['es_hosts']:
+        missing.append("metron-env/es_hosts")
+      if not config['configurations']['metron-env']['es_binary_port']:
+        missing.append("metron-env/es_binary_port")
+      if not config['configurations']['metron-env']['es_date_format']:
+        missing.append("metron-env/es_date_format")
+
+    if len(missing) > 0:
+      raise Fail("Missing required indexing parameters(s): indexer={0}, missing={1}".format(indexer,
missing))


Mime
View raw message