metron-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From o...@apache.org
Subject [26/52] [abbrv] metron git commit: METRON-1462: Separate ES and Kibana from Metron Mpack (mmiklavc via mmiklavc) closes apache/metron#943
Date Wed, 18 Apr 2018 14:59:56 GMT
http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
new file mode 100755
index 0000000..f0903ac
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/dashboardindex.py
@@ -0,0 +1,95 @@
+#!/usr/bin/python
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#  contributor license agreements.  See the NOTICE file distributed with
+#  this work for additional information regarding copyright ownership.
+#  The ASF licenses this file to You under the Apache License, Version 2.0
+#  (the "License"); you may not use this file except in compliance with
+#  the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+from elasticsearch import Elasticsearch
+from elasticsearch.helpers import bulk
+import cPickle as pickle
+import argparse, sys, os.path
+import errno
+import os
+
+
+class DashboardIndex(object):
+
+    def __init__(self, host='localhost', port=9200, url_prefix='', timeout=10, **kwargs):
+        """
+        :arg host: hostname of the node (default: localhost)
+        :arg port: port to use (integer, default: 9200)
+        :arg url_prefix: optional url prefix for elasticsearch
+        :arg timeout: default timeout in seconds (float, default: 10)
+        """
+        self.es = Elasticsearch([{'host':host,'port': port, 'url_prefix': url_prefix, 'timeout':timeout}])
+
+    def get(self):
+        """
+        Get .kibana index from Elasticsearch
+        """
+        dotkibana = self.es.search(index='.kibana', size = 100)
+        return dotkibana['hits']['hits']
+
+    def load(self,filespec):
+        """
+        Save Index data on local filesystem
+        :args filespec: path/filename for saved file
+        """
+        data=[]
+        with open(filespec,'rb') as fp:
+            data = pickle.load(fp)
+        return data
+
+    def save(self,filename,data):
+        """
+        Save Index data on local filesystem
+        :args filespec: path/filename for saved file
+        """
+        with open(filename,'wb') as fp:
+            pickle.dump(data,fp)
+
+    def put(self,data):
+        """
+        Bulk write data to Elasticsearch
+        :args data: data to be written (note: index name is specified in data)
+        """
+        bulk(self.es,data)
+
+    def main(self,args):
+
+        if args.save:
+            print("running save with host:%s on port %d, filespec: %s" % (args.hostname,
args.port, args.filespec))
+            self.save(filename=args.filespec,data=di.get())
+        else:
+            """
+            Loads Kibana Dashboard definition from disk and replaces .kibana on index
+            :args filespec: path/filename for saved file
+            """
+            if not os.path.isfile(args.filespec):
+                raise IOError(
+                    errno.ENOENT, os.strerror(errno.ENOENT), args.filespec)
+            self.es.indices.delete(index='.kibana', ignore=[400, 404])
+            self.put(data=di.load(filespec=args.filespec))
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("hostname", help="ES Hostname or IP", type=str)
+    parser.add_argument("port", help="ES Port", type=int)
+    parser.add_argument("filespec", help="file to be pushed from or saved to", type=str)
+    parser.add_argument("-s","--save", help="run in SAVE mode - .kibana will be read and
saved to filespec",action="store_true")
+    args = parser.parse_args()
+    di = DashboardIndex(host=args.hostname,port=args.port)
+    di.main(args)

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
new file mode 100644
index 0000000..6f38ed5
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/dashboard/kibana.template
@@ -0,0 +1,233 @@
+{
+  "template" : ".kibana",
+    "mappings" : {
+      "search" : {
+        "dynamic" : "strict",
+        "properties" : {
+          "columns" : {
+            "type" : "keyword"
+          },
+          "description" : {
+            "type" : "text"
+          },
+          "hits" : {
+            "type" : "integer"
+          },
+          "kibanaSavedObjectMeta" : {
+            "properties" : {
+              "searchSourceJSON" : {
+                "type" : "text"
+              }
+            }
+          },
+          "sort" : {
+            "type" : "keyword"
+          },
+          "title" : {
+            "type" : "text"
+          },
+          "version" : {
+            "type" : "integer"
+          }
+        }
+      },
+      "url" : {
+        "dynamic" : "strict",
+        "properties" : {
+          "accessCount" : {
+            "type" : "long"
+          },
+          "accessDate" : {
+            "type" : "date"
+          },
+          "createDate" : {
+            "type" : "date"
+          },
+          "url" : {
+            "type" : "text",
+            "fields" : {
+              "keyword" : {
+                "type" : "keyword",
+                "ignore_above" : 2048
+              }
+            }
+          }
+        }
+      },
+      "dashboard" : {
+        "dynamic" : "strict",
+        "properties" : {
+          "description" : {
+            "type" : "text"
+          },
+          "hits" : {
+            "type" : "integer"
+          },
+          "kibanaSavedObjectMeta" : {
+            "properties" : {
+              "searchSourceJSON" : {
+                "type" : "text"
+              }
+            }
+          },
+          "optionsJSON" : {
+            "type" : "text"
+          },
+          "panelsJSON" : {
+            "type" : "text"
+          },
+          "refreshInterval" : {
+            "properties" : {
+              "display" : {
+                "type" : "keyword"
+              },
+              "pause" : {
+                "type" : "boolean"
+              },
+              "section" : {
+                "type" : "integer"
+              },
+              "value" : {
+                "type" : "integer"
+              }
+            }
+          },
+          "timeFrom" : {
+            "type" : "keyword"
+          },
+          "timeRestore" : {
+            "type" : "boolean"
+          },
+          "timeTo" : {
+            "type" : "keyword"
+          },
+          "title" : {
+            "type" : "text"
+          },
+          "uiStateJSON" : {
+            "type" : "text"
+          },
+          "version" : {
+            "type" : "integer"
+          }
+        }
+      },
+      "index-pattern" : {
+        "dynamic" : "strict",
+        "properties" : {
+          "fieldFormatMap" : {
+            "type" : "text"
+          },
+          "fields" : {
+            "type" : "text"
+          },
+          "intervalName" : {
+            "type" : "keyword"
+          },
+          "notExpandable" : {
+            "type" : "boolean"
+          },
+          "sourceFilters" : {
+            "type" : "text"
+          },
+          "timeFieldName" : {
+            "type" : "keyword"
+          },
+          "title" : {
+            "type" : "text"
+          }
+        }
+      },
+      "timelion-sheet" : {
+        "dynamic" : "strict",
+        "properties" : {
+          "description" : {
+            "type" : "text"
+          },
+          "hits" : {
+            "type" : "integer"
+          },
+          "kibanaSavedObjectMeta" : {
+            "properties" : {
+              "searchSourceJSON" : {
+                "type" : "text"
+              }
+            }
+          },
+          "timelion_chart_height" : {
+            "type" : "integer"
+          },
+          "timelion_columns" : {
+            "type" : "integer"
+          },
+          "timelion_interval" : {
+            "type" : "keyword"
+          },
+          "timelion_other_interval" : {
+            "type" : "keyword"
+          },
+          "timelion_rows" : {
+            "type" : "integer"
+          },
+          "timelion_sheet" : {
+            "type" : "text"
+          },
+          "title" : {
+            "type" : "text"
+          },
+          "version" : {
+            "type" : "integer"
+          }
+        }
+      },
+      "visualization" : {
+        "dynamic" : "strict",
+        "properties" : {
+          "description" : {
+            "type" : "text"
+          },
+          "kibanaSavedObjectMeta" : {
+            "properties" : {
+              "searchSourceJSON" : {
+                "type" : "text"
+              }
+            }
+          },
+          "savedSearchId" : {
+            "type" : "keyword"
+          },
+          "title" : {
+            "type" : "text"
+          },
+          "uiStateJSON" : {
+            "type" : "text"
+          },
+          "version" : {
+            "type" : "integer"
+          },
+          "visState" : {
+            "type" : "text"
+          }
+        }
+      },
+      "server" : {
+        "dynamic" : "strict",
+        "properties" : {
+          "uuid" : {
+            "type" : "keyword"
+          }
+        }
+      },
+      "_default_" : {
+        "dynamic" : "strict"
+      },
+      "config" : {
+        "dynamic" : "true",
+        "properties" : {
+          "buildNum" : {
+            "type" : "keyword"
+          }
+        }
+      }
+    }
+}

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index ce8c074..1cd6f4c 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -14,8 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
+import errno
 import os
 import requests
+
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
 from resource_management.core.exceptions import ComponentIsNotRunning
 from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute
@@ -149,6 +153,34 @@ class Indexing(Script):
               cmd.format(params.es_http_url, template_name),
               logoutput=True)
 
+    @OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+    def kibana_dashboard_install(self, env):
+      from params import params
+      env.set_params(params)
+
+      Logger.info("Connecting to Elasticsearch on: %s" % (params.es_http_url))
+
+      kibanaTemplate = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard',
'kibana.template')
+      if not os.path.isfile(kibanaTemplate):
+        raise IOError(
+            errno.ENOENT, os.strerror(errno.ENOENT), kibanaTemplate)
+
+      Logger.info("Loading .kibana index template from %s" % kibanaTemplate)
+      template_cmd = ambari_format(
+          'curl -s -XPOST http://{es_http_url}/_template/.kibana -d @%s' % kibanaTemplate)
+      Execute(template_cmd, logoutput=True)
+
+      kibanaDashboardLoad = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dashboard',
'dashboard-bulkload.json')
+      if not os.path.isfile(kibanaDashboardLoad):
+        raise IOError(
+            errno.ENOENT, os.strerror(errno.ENOENT), kibanaDashboardLoad)
+
+      Logger.info("Loading .kibana dashboard from %s" % kibanaDashboardLoad)
+
+      kibana_cmd = ambari_format(
+          'curl -s -H "Content-Type: application/x-ndjson" -XPOST http://{es_http_url}/.kibana/_bulk
--data-binary @%s' % kibanaDashboardLoad)
+      Execute(kibana_cmd, logoutput=True)
+
     def zeppelin_notebook_import(self, env):
         from params import params
         env.set_params(params)

http://git-wip-us.apache.org/repos/asf/metron/blob/0ab39a32/metron-deployment/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/pom.xml b/metron-deployment/pom.xml
index fe76ade..f59324c 100644
--- a/metron-deployment/pom.xml
+++ b/metron-deployment/pom.xml
@@ -49,6 +49,7 @@
                 <activeByDefault>true</activeByDefault>
             </activation>
             <modules>
+                <module>packaging/ambari/elasticsearch-mpack</module>
                 <module>packaging/ambari/metron-mpack</module>
             </modules>
         </profile>


Mime
View raw message