Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id D0B5E200AE4 for ; Fri, 24 Jun 2016 23:03:46 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id CF089160A58; Fri, 24 Jun 2016 21:03:46 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 070D6160A2E for ; Fri, 24 Jun 2016 23:03:44 +0200 (CEST) Received: (qmail 8855 invoked by uid 500); 24 Jun 2016 21:03:44 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 8841 invoked by uid 99); 24 Jun 2016 21:03:44 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 24 Jun 2016 21:03:44 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 09F16E049D; Fri, 24 Jun 2016 21:03:44 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: oleewere@apache.org To: commits@ambari.apache.org Date: Fri, 24 Jun 2016 21:03:44 -0000 Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: [1/2] ambari git commit: AMBARI-17423. Add missing components to LogSearch Part1 (oleewere) archived-at: Fri, 24 Jun 2016 21:03:47 -0000 Repository: ambari Updated Branches: refs/heads/trunk 36d1f8659 -> 128f77e2f AMBARI-17423. Add missing components to LogSearch Part1 (oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/128f77e2 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/128f77e2 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/128f77e2 Branch: refs/heads/trunk Commit: 128f77e2f12483890064f515ea7691037b520952 Parents: 4207e60 Author: oleewere Authored: Fri Jun 24 17:50:27 2016 +0200 Committer: oleewere Committed: Fri Jun 24 23:00:12 2016 +0200 ---------------------------------------------------------------------- .../src/main/resources/HadoopServiceConfig.json | 467 +++++++++---------- .../AMBARI_METRICS/0.1.0/metainfo.xml | 12 + .../LOGSEARCH/0.5.0/package/scripts/params.py | 18 +- .../0.5.0/package/templates/grok-patterns.j2 | 3 + .../package/templates/input.config-ams.json.j2 | 85 +++- .../package/templates/input.config-hdfs.json.j2 | 21 +- .../package/templates/input.config-hive.json.j2 | 37 +- .../package/templates/input.config-hst.json.j2 | 93 ++++ .../templates/input.config-spark.json.j2 | 66 +++ .../templates/input.config-spark2.json.j2 | 66 +++ .../package/templates/input.config-yarn.json.j2 | 12 +- .../common-services/SPARK/1.2.1/metainfo.xml | 6 + .../common-services/SPARK/1.4.1/metainfo.xml | 6 + .../common-services/SPARK2/2.0.0/metainfo.xml | 12 + .../stacks/HDP/2.5/services/SPARK/metainfo.xml | 6 + .../stacks/2.4/LOGSEARCH/test_logfeeder.py | 4 +- 16 files changed, 651 insertions(+), 263 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json index abe692d..d407d82 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json @@ -1,472 +1,459 @@ { - "service":{ - "accumulo":{ - "label":"Accumulo", - "components":[ + "service": { + "accumulo": { + "label": "Accumulo", + "components": [ { - "name":"accumulo_gc" + "name": "accumulo_gc" }, { - "name":"accumulo_master" + "name": "accumulo_master" }, { - "name":"accumulo_monitor" + "name": "accumulo_monitor" }, { - "name":"accumulo_tracer" + "name": "accumulo_tracer" }, { - "name":"accumulo_tserver" + "name": "accumulo_tserver" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "atlas":{ - "label":"Atlas", - "components":[ + "atlas": { + "label": "Atlas", + "components": [ { - "name":"atlas_app" + "name": "atlas_app" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "ambari":{ - "label":"Ambari", - "components":[ + "ambari": { + "label": "Ambari", + "components": [ { - "name":"ambari_agent" + "name": "ambari_agent" }, { - "name":"ambari_server" + "name": "ambari_server" }, { - "name":"ambari_alerts" + "name": "ambari_alerts" }, { - "name":"ambari_audit" + "name": "ambari_audit" }, { - "name":"ambari_config_changes" + "name": "ambari_config_changes" }, { - "name":"ambari_eclipseling" + "name": "ambari_eclipselink" }, { - "name":"ambari_server_check_database" + "name": "ambari_server_check_database" } ], - "dependencies":[ - + "dependencies": [ ] - }, - "ams":{ - "label":"AMS", - "components":[ + "ams": { + "label": "AMS", + "components": [ { - "name":"ams_hbase_master" + "name": "ams_hbase_master" }, { - "name":"ams_hbase_regionserver" + "name": "ams_hbase_regionserver" }, { - "name":"ams_collector" + "name": "ams_collector" + }, + { + "name": "ams_monitor" + }, + { + "name": "ams_grafana" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "falcon":{ - "label":"Falcon", - "components":[ + "falcon": { + "label": "Falcon", + "components": [ { - "name":"falcon_app" + "name": "falcon_app" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "flume":{ - "label":"Flume", - "components":[ + "flume": { + "label": "Flume", + "components": [ { - "name":"flume_handler" + "name": "flume_handler" } ], - "dependencies":[ - + "dependencies": [ ] - }, - "hbase":{ - "label":"HBase", - "components":[ + "hbase": { + "label": "HBase", + "components": [ { - "name":"hbase_master" + "name": "hbase_master" }, { - "name":"hbase_regionserver" + "name": "hbase_regionserver" }, { - "name":"hbase_phoenix_server" + "name": "hbase_phoenix_server" } - ], - "dependencies":[ + "dependencies": [ { - "service":"hdfs", - "components":[ + "service": "hdfs", + "components": [ "hdfs_namenode" ] - } - ] - }, - "hdfs":{ - "label":"HDFS", - "components":[ + "hdfs": { + "label": "HDFS", + "components": [ { - "name":"hdfs_datanode" + "name": "hdfs_datanode" }, { - "name":"hdfs_namenode" + "name": "hdfs_namenode" }, { - "name":"hdfs_journalnode" + "name": "hdfs_journalnode" }, { - "name":"hdfs_secondarynamenode" + "name": "hdfs_secondarynamenode" }, { - "name":"hdfs_zkfc" + "name": "hdfs_zkfc" }, { - "name":"hdfs_audit", - "rowtype":"audit" + "name": "hdfs_nfs3" + }, + { + "name": "hdfs_audit", + "rowtype": "audit" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "hive":{ - "label":"Hive", - "components":[ + "hive": { + "label": "Hive", + "components": [ + { + "name": "hive_hiveserver2" + }, { - "name":"hive_hiveserver2" + "name": "hive_metastore" }, { - "name":"hive_metastore" + "name": "webhcat_server" } - ], - "dependencies":[ + "dependencies": [ { - "service":"hdfs", - "components":[ + "service": "hdfs", + "components": [ "hdfs_namenode" ] - } - ] - }, - "kafka":{ - "label":"Kafka", - "components":[ + "kafka": { + "label": "Kafka", + "components": [ { - "name":"kafka_controller" + "name": "kafka_controller" }, { - "name":"kafka_request" + "name": "kafka_request" }, { - "name":"kafka_logcleaner" + "name": "kafka_logcleaner" }, { - "name":"kafka_server" + "name": "kafka_server" }, { - "name":"kafka_statechange" + "name": "kafka_statechange" } - ], - "dependencies":[ + "dependencies": [ { - "service":"zookeeper", - "components":[ + "service": "zookeeper", + "components": [ "zookeeper" ] - } - ] - }, - "knox":{ - "label":"Knox", - "components":[ + "knox": { + "label": "Knox", + "components": [ { - "name":"knox_gateway" + "name": "knox_gateway" }, { - "name":"knox_cli" + "name": "knox_cli" }, { - "name":"knox_ldap" + "name": "knox_ldap" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "nifi":{ - "label":"NiFi", - "components":[ + "nifi": { + "label": "NiFi", + "components": [ { - "name":"nifi_app" + "name": "nifi_app" }, { - "name":"nifi_bootstrap" + "name": "nifi_bootstrap" }, { - "name":"nifi_setup" + "name": "nifi_setup" }, { - "name":"nifi_user" + "name": "nifi_user" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "mapred":{ - "label":"MapReduce", - "components":[ + "mapred": { + "label": "MapReduce", + "components": [ { - "name":"mapred_historyserver" + "name": "mapred_historyserver" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "logsearch":{ - "label":"Logsearch", - "components":[ + "logsearch": { + "label": "Logsearch", + "components": [ { - "name":"logsearch_app" + "name": "logsearch_app" }, { - "name":"logsearch_feeder" + "name": "logsearch_feeder" }, { - "name":"logsearch_perf" + "name": "logsearch_perf" }, { - "name":"logsearch_solr" + "name": "logsearch_solr" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "ranger":{ - "label":"Ranger", - "components":[ + "ranger": { + "label": "Ranger", + "components": [ { - "name":"ranger_admin" + "name": "ranger_admin" }, { - "name":"ranger_dbpatch" + "name": "ranger_dbpatch" }, { - "name":"ranger_kms" + "name": "ranger_kms" }, { - "name":"ranger_usersync" + "name": "ranger_usersync" } - ], - "dependencies":[ + "dependencies": [ { - "service":"hdfs", - "required":"optional", - "components":[ + "service": "hdfs", + "required": "optional", + "components": [ "hdfs_namenode" ] - }, { - "service":"hbase", - "required":"optional", - "components":[ + "service": "hbase", + "required": "optional", + "components": [ "hbase_master", "hbase_regionserver" ] - }, { - "service":"hive", - "required":"optional", - "components":[ + "service": "hive", + "required": "optional", + "components": [ "hive_hiveserver2" ] - }, { - "service":"kafka", - "required":"optional", - "components":[ + "service": "kafka", + "required": "optional", + "components": [ "kafka_ranger" ] - }, { - "service":"knox", - "required":"optional", - "components":[ + "service": "knox", + "required": "optional", + "components": [ "knox_gateway" ] - }, { - "service":"storm", - "required":"optional", - "components":[ + "service": "storm", + "required": "optional", + "components": [ "storm_supervisor" ] - }, { - "service":"yarn", - "required":"optional", - "components":[ + "service": "yarn", + "required": "optional", + "components": [ "yarn_resourcemanager" ] - } - ] - }, - "oozie":{ - "label":"Oozie", - "components":[ + "oozie": { + "label": "Oozie", + "components": [ { - "name":"oozie_app" + "name": "oozie_app" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "yarn":{ - "label":"YARN", - "components":[ + "hst": { + "label": "SmartSense", + "components": [ { - "name":"yarn_nodemanager" + "name": "hst_server" }, { - "name":"yarn_resourcemanager" + "name": "hst_agent" + } + ], + "dependencies": [ + ] + }, + "spark": { + "label": "Spark", + "components": [ + { + "name": "spark_jobhistory_server" }, { - "name":"yarn_timelineserver" + "name": "spark_thriftserver" }, { - "name":"yarn_historyserver" + "name": "livy_server" + } + ], + "dependencies": [ + ] + }, + "spark2": { + "label": "Spark 2", + "components": [ + { + "name": "spark2_jobhistory_server" }, { - "name":"yarn_jobsummary" + "name": "spark2_thriftserver" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "storm":{ - "label":"Storm", - "components":[ + "storm": { + "label": "Storm", + "components": [ + { + "name": "storm_drpc" + }, + { + "name": "storm_logviewer" + }, + { + "name": "storm_nimbus" + }, { - "name":"storm_drpc" + "name": "storm_supervisor" }, { - "name":"storm_logviewer" + "name": "storm_ui" + }, + { + "name": "storm_worker" + } + ], + "dependencies": [ + ] + }, + "yarn": { + "label": "YARN", + "components": [ + { + "name": "yarn_nodemanager" }, { - "name":"storm_nimbus" + "name": "yarn_resourcemanager" }, { - "name":"storm_supervisor" + "name": "yarn_timelineserver" }, { - "name":"storm_ui" + "name": "yarn_historyserver" }, { - "name":"storm_worker" + "name": "yarn_jobsummary" } - ], - "dependencies":[ - + "dependencies": [ ] - }, - "zeppelin":{ - "label":"Zeppelin", - "components":[ + "zeppelin": { + "label": "Zeppelin", + "components": [ { - "name":"zeppelin" + "name": "zeppelin" } ], - "dependencies":[ - + "dependencies": [ ] }, - "zookeeper":{ - "label":"ZooKeeper", - "components":[ + "zookeeper": { + "label": "ZooKeeper", + "components": [ { - "name":"zookeeper" + "name": "zookeeper" } - ], - "dependencies":[ - + "dependencies": [ ] - } - } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml index 4f6979b..08b1a9d 100644 --- a/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/AMBARI_METRICS/0.1.0/metainfo.xml @@ -87,6 +87,12 @@ PYTHON 1200 + + + ams_monitor + true + + @@ -105,6 +111,12 @@ ams-grafana-ini + + + ams_grafana + true + + http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py index 4bc1004..23a9c09 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py @@ -240,9 +240,11 @@ logsearch_app_log4j_content = config['configurations']['logsearch-log4j']['conte ambari_server_log_dir = '/var/log/ambari-server' ambari_agent_log_dir = '/var/log/ambari-agent' knox_log_dir = '/var/log/knox' +hst_log_dir = '/var/log/hst' -metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log') -metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log') +metrics_collector_log_dir = default('/configurations/ams-env/metrics_collector_log_dir', '/var/log/ambari-metrics-collector') +metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_dir', '/var/log/ambari-metrics-monitor') +metrics_grafana_log_dir = default('/configurations/ams-grafana-env/metrics_grafana_log_dir', '/var/log/ambari-metrics-grafana') atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas') accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo') @@ -251,6 +253,7 @@ flume_log_dir = default('/configurations/flume-env/flume_log_dir', '/var/log/flu hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase') hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop') hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive') +hcat_log_dir = default('configurations/hive-env/hcat_log_dir', '/var/log/webhcat') kafka_log_dir = default('/configurations/kafka-env/kafka_log_dir', '/var/log/kafka') nifi_log_dir = default('/configurations/nifi-env/nifi_node_log_dir', '/var/log/nifi') oozie_log_dir = default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie') @@ -262,6 +265,13 @@ yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/ mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop') zeppelin_log_dir = default('/configuration/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin') zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper') +spark_log_dir = default('/configurations/spark-env/spark_log_dir', '/var/log/spark') +livy_log_dir = default('/configurations/livy-env/livy_log_dir', '/var/log/livy') +spark2_log_dir = default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2') + +hdfs_user = default('configurations/hadoop-env/hdfs_user', 'hdfs') +mapred_user = default('configurations/mapred-env/mapred_user', 'mapred') +yarn_user = default('configurations/yarn-env/yarn_user', 'yarn') ##################################### # Logsearch auth configs @@ -330,8 +340,8 @@ logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.c logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable', True)).lower() logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval', 5) -logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka', - 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper'] +logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'kafka', + 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper'] logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 index c85a06a..874c59d 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/grok-patterns.j2 @@ -142,3 +142,6 @@ LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo| # Custom USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} +SPARK_DATESTAMP %{YEAR}[/-]%{MONTHNUM2}[/-]%{MONTHDAY} %{TIME} +CUSTOM_DATESTAMP %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} +CUSTOM_SEPARATOR %{SPACE}\|%{SPACE} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 index 34c1ed4..3dbe6b2 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-ams.json.j2 @@ -31,8 +31,17 @@ "type":"ams_collector", "rowtype":"service", "path":"{{metrics_collector_log_dir}}/ambari-metrics-collector.log" + }, + { + "type":"ams_monitor", + "rowtype":"service", + "path":"{{metrics_monitor_log_dir}}/ambari-metrics-monitor.out" + }, + { + "type":"ams_grafana", + "rowtype":"service", + "path":"{{metrics_grafana_log_dir}}/grafana.log" } - ], "filter":[ { @@ -85,7 +94,79 @@ } - } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_grafana" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern":"^(%{DATESTAMP})", + "message_pattern":"(?m)^%{DATESTAMP}%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy/MM/dd HH:mm:ss" + } + + }, + "level":[ + { + "map_fieldvalue":{ + "pre_value":"I", + "post_value":"INFO" + }, + "map_fieldvalue":{ + "pre_value":"W", + "post_value":"WARN" + }, + "map_fieldvalue":{ + "pre_value":"D", + "post_value":"DEBUG" + }, + "map_fieldvalue":{ + "pre_value":"E", + "post_value":"ERROR" + }, + "map_fieldvalue":{ + "pre_value":"F", + "post_value":"FATAL" + } + } + ] + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ams_monitor" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{DATESTAMP:logtime})", + "message_pattern":"(?m)^%{DATESTAMP:logtime}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 index 730ef99..13ff9d1 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hdfs.json.j2 @@ -20,27 +20,32 @@ { "type":"hdfs_datanode", "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log" + "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-datanode-*.log" }, { "type":"hdfs_namenode", "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log" + "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-namenode-*.log" }, { "type":"hdfs_journalnode", "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log" + "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-journalnode-*.log" }, { "type":"hdfs_secondarynamenode", "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log" + "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-secondarynamenode-*.log" }, { "type":"hdfs_zkfc", "rowtype":"service", - "path":"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log" + "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hadoop-{{hdfs_user}}-zkfc-*.log" + }, + { + "type":"hdfs_nfs3", + "rowtype":"service", + "path":"{{hdfs_log_dir_prefix}}/{{hdfs_user}}/hadoop-{{hdfs_user}}-nfs3-*.log" }, { "type":"hdfs_audit", @@ -52,7 +57,7 @@ "repoType":"1", "repo":"hdfs" }, - "path":"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log" + "path":"{{hdfs_log_dir_prefix}}/{hdfs_user}}/hdfs-audit.log" } ], @@ -66,7 +71,8 @@ "hdfs_journalnode", "hdfs_secondarynamenode", "hdfs_namenode", - "hdfs_zkfc" + "hdfs_zkfc", + "hdfs_nfs3" ] } @@ -80,7 +86,6 @@ "map_date":{ "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" } - } } http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 index 131ae86..f0fbf0a 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hive.json.j2 @@ -26,8 +26,12 @@ "type":"hive_metastore", "rowtype":"service", "path":"{{hive_log_dir}}/hivemetastore.log" + }, + { + "type": "webhcat_server", + "rowntype":"service", + "path":"{{hcat_log_dir}}/webhcat.log" } - ], "filter":[ { @@ -55,6 +59,37 @@ } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "webhcat_server" + ] + + } + + }, + "log4j_format":" %-5p | %d{DATE} | %c | %m%n", + "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime})", + "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{CUSTOM_SEPARATOR}%{CUSTOM_DATESTAMP:logtime}%{CUSTOM_SEPARATOR}%{JAVACLASS:file}%{CUSTOM_SEPARATOR}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss,SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + + } + } ] http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2 new file mode 100644 index 0000000..ee19f14 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hst.json.j2 @@ -0,0 +1,93 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} + { + "input":[ + + { + "type":"hst_server", + "rowtype":"service", + "path":"{{hst_log_dir}}/hst-server.log" + }, + { + "type":"hst_agent", + "rowtype":"service", + "path":"{{hst_log_dir}}/hst-agent.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hst_agent" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{LOGLEVEL:level}%{SPACE}%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{LOGLEVEL:level}%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVAFILE:file}%{GREEDYDATA:log_message}}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "hst_server" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{CUSTOM_DATESTAMP:logtime})", + "message_pattern":"(?m)^%{CUSTOM_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVAFILE:file}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"dd MMM yyyy HH:mm:ss,SSS" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + + } + } + + ] + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 new file mode 100644 index 0000000..80be6ee --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark.json.j2 @@ -0,0 +1,66 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} + { + "input":[ + + { + "type":"spark_jobhistory_server", + "rowtype":"service", + "path":"{{spark_log_dir}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" + }, + { + "type":"spark_thriftserver", + "rowtype":"service", + "path":"{{spark_log_dir}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "spark_jobhistory_server", + "spark_thriftserver" + ] + + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", + "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yy/MM/dd HH:mm:ss" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + } + ] + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 new file mode 100644 index 0000000..20bd946 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-spark2.json.j2 @@ -0,0 +1,66 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} + { + "input":[ + + { + "type":"spark2_jobhistory_server", + "rowtype":"service", + "path":"{{spark2_log_dir}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" + }, + { + "type":"spark2_thriftserver", + "rowtype":"service", + "path":"{{spark2_log_dir}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "spark2_jobhistory_server", + "spark2_thriftserver" + ] + + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", + "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "date_pattern":"yy/MM/dd HH:mm:ss" + } + + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + } + ] + + } http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 index fb35620..3d9677e 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-yarn.json.j2 @@ -20,32 +20,32 @@ { "type":"mapred_historyserver", "rowtype":"service", - "path":"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log" + "path":"{{mapred_log_dir_prefix}}/{{mapred_user}}/mapred-{{mapred_user}}-historyserver*.log" }, { "type":"yarn_nodemanager", "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log" + "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-nodemanager-*.log" }, { "type":"yarn_resourcemanager", "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log" + "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-resourcemanager-*.log" }, { "type":"yarn_timelineserver", "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log" + "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-timelineserver-*.log" }, { "type":"yarn_historyserver", "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log" + "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/yarn-{{yarn_user}}-historyserver-*.log" }, { "type":"yarn_jobsummary", "rowtype":"service", - "path":"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log" + "path":"{{yarn_log_dir_prefix}}/{{yarn_user}}/hadoop-mapreduce.jobsummary.log" } ], http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml index 86af492..1a35793 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/metainfo.xml @@ -61,6 +61,12 @@ + + + spark_jobhistory_server + true + + PYTHON http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml index 99bab04..bc9f7a4 100644 --- a/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SPARK/1.4.1/metainfo.xml @@ -62,6 +62,12 @@ + + + spark_thriftserver + true + + PYTHON http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml index 783f049..c83c83c 100755 --- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/metainfo.xml @@ -59,6 +59,12 @@ PYTHON 600 + + + spark2_jobhistory_server + true + + SPARK2_THRIFTSERVER @@ -101,6 +107,12 @@ PYTHON 600 + + + spark2_thriftserver + true + + SPARK2_CLIENT http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml index 67303b6..8b14fd1 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/SPARK/metainfo.xml @@ -59,6 +59,12 @@ PYTHON 600 + + + livy_server + true + + http://git-wip-us.apache.org/repos/asf/ambari/blob/128f77e2/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index 26e8934..63afcac 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -98,8 +98,8 @@ class TestLogFeeder(RMFTestCase): encoding='utf-8' ) - logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'kafka', - 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm', 'yarn', 'zeppelin', 'zookeeper'] + logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', 'kafka', + 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', 'yarn', 'zeppelin', 'zookeeper'] logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services]