metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [28/52] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Date Wed, 18 Apr 2018 14:59:58 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
deleted file mode 100755
index f0903ac..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/dashboardindex.py
+++ /dev/null
@@ -1,95 +0,0 @@
-#!/usr/bin/python
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#  contributor license agreements.  See the NOTICE file distributed with
-#  this work for additional information regarding copyright ownership.
-#  The ASF licenses this file to You under the Apache License, Version 2.0
-#  (the "License"); you may not use this file except in compliance with
-#  the License.  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.
-#
-
-from elasticsearch import Elasticsearch
-from elasticsearch.helpers import bulk
-import cPickle as pickle
-import argparse, sys, os.path
-import errno
-import os
-
-
-class DashboardIndex(object):
-
-    def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
-        """
-        :arg host: hostname of the node (default: localhost)
-        :arg port: port to use (integer, default: 9200)
-        :arg url_prefix: optional url prefix for elasticsearch
-        :arg timeout: default timeout in seconds (float, default: 10)
-        """
-        self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
-
-    def get(self):
-        """
-        Get .kibana index from Elasticsearch
-        """
-        dotkibana = self.es.search(index='.kibana', size = 100)
-        return dotkibana['hits']['hits']
-
-    def load(self,filespec):
-        """
-        Save Index data on local filesystem
-        :args filespec: path/filename for saved file
-        """
-        data=[]
-        with open(filespec,'rb') as fp:
-            data = pickle.load(fp)
-        return data
-
-    def save(self,filename,data):
-        """
-        Save Index data on local filesystem
-        :args filespec: path/filename for saved file
-        """
-        with open(filename,'wb') as fp:
-            pickle.dump(data,fp)
-
-    def put(self,data):
-        """
-        Bulk write data to Elasticsearch
-        :args data: data to be written (note: index name is specified in data)
-        """
-        bulk(self.es,data)
-
-    def main(self,args):
-
-        if args.save:
-            print("running save with host:%s on port %d, filespec: %s" % (args.hostname,
args.port, args.filespec))
-            self.save(filename=args.filespec,data=di.get())
-        else:
-            """
-            Loads Kibana Dashboard definition from disk and replaces .kibana on index
-            :args filespec: path/filename for saved file
-            """
-            if not os.path.isfile(args.filespec):
-                raise IOError(
-                    errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
-            self.es.indices.delete(index='.kibana', ignore=[400, 404])
-            self.put(data=di.load(filespec=args.filespec))
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser()
-    parser.add_argument("hostname", help="ES Hostname or IP", type=str)
-    parser.add_argument("port", help="ES Port", type=int)
-    parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
-    parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and
saved to filespec",action="store_true")
-    args = parser.parse_args()
-    di = DashboardIndex(host=args.hostname,port=args.port)
-    di.main(args)

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
deleted file mode 100644
index 6f38ed5..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/dashboard/kibana.template
+++ /dev/null
@@ -1,233 +0,0 @@
-{
-  "template" : ".kibana",
-    "mappings" : {
-      "search" : {
-        "dynamic" : "strict",
-        "properties" : {
-          "columns" : {
-            "type" : "keyword"
-          },
-          "description" : {
-            "type" : "text"
-          },
-          "hits" : {
-            "type" : "integer"
-          },
-          "kibanaSavedObjectMeta" : {
-            "properties" : {
-              "searchSourceJSON" : {
-                "type" : "text"
-              }
-            }
-          },
-          "sort" : {
-            "type" : "keyword"
-          },
-          "title" : {
-            "type" : "text"
-          },
-          "version" : {
-            "type" : "integer"
-          }
-        }
-      },
-      "url" : {
-        "dynamic" : "strict",
-        "properties" : {
-          "accessCount" : {
-            "type" : "long"
-          },
-          "accessDate" : {
-            "type" : "date"
-          },
-          "createDate" : {
-            "type" : "date"
-          },
-          "url" : {
-            "type" : "text",
-            "fields" : {
-              "keyword" : {
-                "type" : "keyword",
-                "ignore_above" : 2048
-              }
-            }
-          }
-        }
-      },
-      "dashboard" : {
-        "dynamic" : "strict",
-        "properties" : {
-          "description" : {
-            "type" : "text"
-          },
-          "hits" : {
-            "type" : "integer"
-          },
-          "kibanaSavedObjectMeta" : {
-            "properties" : {
-              "searchSourceJSON" : {
-                "type" : "text"
-              }
-            }
-          },
-          "optionsJSON" : {
-            "type" : "text"
-          },
-          "panelsJSON" : {
-            "type" : "text"
-          },
-          "refreshInterval" : {
-            "properties" : {
-              "display" : {
-                "type" : "keyword"
-              },
-              "pause" : {
-                "type" : "boolean"
-              },
-              "section" : {
-                "type" : "integer"
-              },
-              "value" : {
-                "type" : "integer"
-              }
-            }
-          },
-          "timeFrom" : {
-            "type" : "keyword"
-          },
-          "timeRestore" : {
-            "type" : "boolean"
-          },
-          "timeTo" : {
-            "type" : "keyword"
-          },
-          "title" : {
-            "type" : "text"
-          },
-          "uiStateJSON" : {
-            "type" : "text"
-          },
-          "version" : {
-            "type" : "integer"
-          }
-        }
-      },
-      "index-pattern" : {
-        "dynamic" : "strict",
-        "properties" : {
-          "fieldFormatMap" : {
-            "type" : "text"
-          },
-          "fields" : {
-            "type" : "text"
-          },
-          "intervalName" : {
-            "type" : "keyword"
-          },
-          "notExpandable" : {
-            "type" : "boolean"
-          },
-          "sourceFilters" : {
-            "type" : "text"
-          },
-          "timeFieldName" : {
-            "type" : "keyword"
-          },
-          "title" : {
-            "type" : "text"
-          }
-        }
-      },
-      "timelion-sheet" : {
-        "dynamic" : "strict",
-        "properties" : {
-          "description" : {
-            "type" : "text"
-          },
-          "hits" : {
-            "type" : "integer"
-          },
-          "kibanaSavedObjectMeta" : {
-            "properties" : {
-              "searchSourceJSON" : {
-                "type" : "text"
-              }
-            }
-          },
-          "timelion_chart_height" : {
-            "type" : "integer"
-          },
-          "timelion_columns" : {
-            "type" : "integer"
-          },
-          "timelion_interval" : {
-            "type" : "keyword"
-          },
-          "timelion_other_interval" : {
-            "type" : "keyword"
-          },
-          "timelion_rows" : {
-            "type" : "integer"
-          },
-          "timelion_sheet" : {
-            "type" : "text"
-          },
-          "title" : {
-            "type" : "text"
-          },
-          "version" : {
-            "type" : "integer"
-          }
-        }
-      },
-      "visualization" : {
-        "dynamic" : "strict",
-        "properties" : {
-          "description" : {
-            "type" : "text"
-          },
-          "kibanaSavedObjectMeta" : {
-            "properties" : {
-              "searchSourceJSON" : {
-                "type" : "text"
-              }
-            }
-          },
-          "savedSearchId" : {
-            "type" : "keyword"
-          },
-          "title" : {
-            "type" : "text"
-          },
-          "uiStateJSON" : {
-            "type" : "text"
-          },
-          "version" : {
-            "type" : "integer"
-          },
-          "visState" : {
-            "type" : "text"
-          }
-        }
-      },
-      "server" : {
-        "dynamic" : "strict",
-        "properties" : {
-          "uuid" : {
-            "type" : "keyword"
-          }
-        }
-      },
-      "_default_" : {
-        "dynamic" : "strict"
-      },
-      "config" : {
-        "dynamic" : "true",
-        "properties" : {
-          "buildNum" : {
-            "type" : "keyword"
-          }
-        }
-      }
-    }
-}

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
deleted file mode 100755
index a15f709..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
+++ /dev/null
@@ -1,119 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-import errno
-import os
-
-from ambari_commons.os_check import OSCheck
-from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-
-from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import Execute
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.libraries.functions.format import format as ambari_format
-from resource_management.libraries.script import Script
-
-from common import service_check
-
-class Kibana(Script):
-
-    def install(self, env):
-        import params
-        env.set_params(params)
-        Logger.info("Installing Kibana")
-        self.install_packages(env)
-
-    def configure(self, env, upgrade_type=None, config_dir=None):
-        import params
-        env.set_params(params)
-        Logger.info("Configuring Kibana")
-
-        directories = [params.log_dir, params.pid_dir, params.conf_dir]
-        Directory(directories,
-                  mode=0755,
-                  owner=params.kibana_user,
-                  group=params.kibana_user
-                  )
-
-        File("{0}/kibana.yml".format(params.conf_dir),
-             owner=params.kibana_user,
-             content=InlineTemplate(params.kibana_yml_template)
-             )
-
-    def stop(self, env, upgrade_type=None):
-        import params
-        env.set_params(params)
-        Logger.info("Stopping Kibana")
-        Execute("service kibana stop")
-
-    def start(self, env, upgrade_type=None):
-        import params
-        env.set_params(params)
-        self.configure(env)
-        Logger.info("Starting Kibana")
-        Execute("service kibana start")
-
-    def restart(self, env):
-        import params
-        env.set_params(params)
-        self.configure(env)
-        Logger.info("Restarting Kibana")
-        Execute("service kibana restart")
-
-    def status(self, env):
-        import params
-        env.set_params(params)
-        Logger.info('Status check Kibana')
-        service_check("service kibana status", user=params.kibana_user, label="Kibana")
-
-    @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
-    def load_template(self, env):
-        import params
-        env.set_params(params)
-
-        hostname = ambari_format("{es_host}")
-        port = int(ambari_format("{es_port}"))
-
-        Logger.info("Connecting to Elasticsearch on host: %s, port: %s" % (hostname, port))
-
-        kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard',
'kibana.template')
-        if not os.path.isfile(kibanaTemplate):
-          raise IOError(
-              errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate)
-
-        Logger.info("Loading .kibana index template from %s" % kibanaTemplate)
-        template_cmd = ambari_format(
-            'curl -s -XPOST http://{es_host}:{es_port}/_template/.kibana -d @%s' % kibanaTemplate)
-        Execute(template_cmd, logoutput=True)
-
-        kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard',
'dashboard-bulkload.json')
-        if not os.path.isfile(kibanaDashboardLoad):
-          raise IOError(
-              errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad)
-
-        Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad)
-
-        kibana_cmd = ambari_format(
-            'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_host}:{es_port}/.kibana/_bulk
--data-binary @%s' % kibanaDashboardLoad)
-        Execute(kibana_cmd, logoutput=True)
-
-
-if __name__ == "__main__":
-    Kibana().execute()

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
deleted file mode 100755
index ef4cb62..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/params.py
+++ /dev/null
@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements.  See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership.  The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License.  You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Kibana Params configurations
-
-"""
-
-from urlparse import urlparse
-
-from resource_management.libraries.functions import format
-from resource_management.libraries.script import Script
-
-# server configurations
-config = Script.get_config()
-
-kibana_home = '/usr/share/kibana/'
-kibana_bin = '/usr/share/kibana/bin/'
-
-conf_dir = "/etc/kibana"
-kibana_user = config['configurations']['kibana-env']['kibana_user']
-kibana_group = config['configurations']['kibana-env']['kibana_group']
-log_dir = config['configurations']['kibana-env']['kibana_log_dir']
-pid_dir = config['configurations']['kibana-env']['kibana_pid_dir']
-pid_file = format("{pid_dir}/kibanasearch.pid")
-es_url = config['configurations']['kibana-env']['kibana_es_url']
-parsed = urlparse(es_url)
-es_host = parsed.netloc.split(':')[0]
-es_port = parsed.netloc.split(':')[1]
-kibana_port = config['configurations']['kibana-env']['kibana_server_port']
-kibana_server_host = config['configurations']['kibana-env']['kibana_server_host']
-kibana_default_application = config['configurations']['kibana-env']['kibana_default_application']
-hostname = config['hostname']
-java64_home = config['hostLevelParams']['java_home']
-kibana_yml_template = config['configurations']['kibana-site']['content']
-

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
deleted file mode 100755
index 448e102..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/quicklinks/quicklinks.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-  "name": "default",
-  "description": "default quick links configuration",
-  "configuration": {
-    "protocol":
-    {
-      "type":"HTTP_ONLY"
-    },
-
-    "links": [
-      {
-        "name": "metron_ui",
-        "label": "Metron UI",
-        "requires_user_name": "false",
-        "component_name": "KIBANA_MASTER",
-        "url":"%@://%@:%@/",
-        "port":{
-          "http_property": "kibana_server_port",
-          "http_default_port": "5601",
-          "https_property": "kibana_server_port",
-          "https_default_port": "5601",
-          "regex": "^(\\d+)$",
-          "site": "kibana-env"
-        }
-      }
-    ]
-  }
-}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
index 6a20af8..38bd94a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/metainfo.xml
@@ -211,6 +211,15 @@
                 </commandScript>
               </customCommand>
               <customCommand>
+                <name>KIBANA_DASHBOARD_INSTALL</name>
+                <background>false</background>
+                <commandScript>
+                  <script>scripts/indexing_master.py</script>
+                  <scriptType>PYTHON</scriptType>
+                  <timeout>600</timeout>
+                </commandScript>
+              </customCommand>
+              <customCommand>
                 <name>ZEPPELIN_NOTEBOOK_IMPORT</name>
                 <commandScript>
                   <script>scripts/indexing_master.py</script>

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
new file mode 100755
index 0000000..8d2bad8
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/__init__.py
@@ -0,0 +1,16 @@
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
\ No newline at end of file


Mime
View raw message