Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4833A200BA5 for ; Wed, 19 Oct 2016 17:57:02 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 47198160AFB; Wed, 19 Oct 2016 15:57:02 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 503B7160AEA for ; Wed, 19 Oct 2016 17:57:00 +0200 (CEST) Received: (qmail 46348 invoked by uid 500); 19 Oct 2016 15:56:59 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 46181 invoked by uid 99); 19 Oct 2016 15:56:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 19 Oct 2016 15:56:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 3C1DCE390E; Wed, 19 Oct 2016 15:56:59 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: oleewere@apache.org To: commits@ambari.apache.org Date: Wed, 19 Oct 2016 15:57:02 -0000 Message-Id: <38b99fabfa4a451e846e16993e5348f5@git.apache.org> In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [4/6] ambari git commit: AMBARI-18548. Declarative Logsearch/Logfeeder Component Metadata for Stack Component (oleewere) archived-at: Wed, 19 Oct 2016 15:57:02 -0000 http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml new file mode 100644 index 0000000..2f13d3f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/configuration/oozie-logsearch-conf.xml @@ -0,0 +1,80 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + Oozie + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + OOZIE_SERVER:oozie_app + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"oozie_app", + "rowtype":"service", + "path":"{{default('/configurations/oozie-env/oozie_log_dir', '/var/log/oozie')}}/oozie.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "oozie_app" + ] + } + }, + "log4j_format":"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml new file mode 100644 index 0000000..7b7421d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER/0.4.0/configuration/ranger-logsearch-conf.xml @@ -0,0 +1,111 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + Ranger + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + RANGER_SERVER:ranger_server,ranger_dbpatch;RANGER_USERSYNC:ranger_usersync; + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"ranger_admin", + "rowtype":"service", + "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/xa_portal.log" + }, + { + "type":"ranger_dbpatch", + "is_enabled":"true", + "path":"{{default('/configurations/ranger-env/ranger_admin_log_dir', '/var/log/ranger/admin')}}/ranger_db_patch.log" + }, + { + "type":"ranger_usersync", + "rowtype":"service", + "path":"{{default('/configurations/ranger-env/ranger_usersync_log_dir', '/var/log/ranger/usersync')}}/usersync.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_admin", + "ranger_dbpatch" + ] + } + }, + "log4j_format":"%d [%t] %-5p %C{6} (%F:%L) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\$%{JAVAFILE:file}:%{INT:line_number}\$%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_usersync" + ] + } + }, + "log4j_format":"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n", + "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", + "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"dd MMM yyyy HH:mm:ss" + } + } + } + } + ] + } + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml new file mode 100644 index 0000000..226d2b0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/RANGER_KMS/0.5.0.2.3/configuration/ranger-kms-logsearch-conf.xml @@ -0,0 +1,80 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + Ranger KMS + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + RANGER_KMS:ranger_kms + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"ranger_kms", + "rowtype":"service", + "path":"{{default('/configurations/kms-env/kms_log_dir', '/var/log/ranger/kms')}}/kms.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ranger_kms" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{1} - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml new file mode 100644 index 0000000..63201ef --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.1/configuration/spark-logsearch-conf.xml @@ -0,0 +1,98 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + Spark + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + SPARK_JOBHISTORYSERVER:spark_jobhistory_server;SPARK_THRIFTSERVER:spark_thriftserver;LIVY_SERVER:livy_server + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"spark_jobhistory_server", + "rowtype":"service", + "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" + }, + { + "type":"spark_thriftserver", + "rowtype":"service", + "path":"{{default('/configurations/spark-env/spark_log_dir', '/var/log/spark')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + }, + { + "type":"livy_server", + "rowtype":"service", + "path":"{{default('/configurations/livy-env/livy_log_dir', '/var/log/livy')}}/livy-livy-server.out" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "spark_jobhistory_server", + "spark_thriftserver", + "livy_server" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", + "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yy/MM/dd HH:mm:ss" + } + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + } + } + ] +} + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml new file mode 100644 index 0000000..3b00d51 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/configuration/spark2-logsearch-conf.xml @@ -0,0 +1,92 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + Spark2 + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + SPARK2_JOBHISTORYSERVER:spark2_jobhistory_server;SPARK2_THRIFTSERVER:spark2_thriftserver + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"spark2_jobhistory_server", + "rowtype":"service", + "path":"{{default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')}}/spark-*-org.apache.spark.deploy.history.HistoryServer*.out" + }, + { + "type":"spark2_thriftserver", + "rowtype":"service", + "path":"{{default('/configurations/spark2-env/spark_log_dir', '/var/log/spark2')}}/spark-*-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2*.out" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "spark2_jobhistory_server", + "spark2_thriftserver" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level})", + "message_pattern":"(?m)^%{SPARK_DATESTAMP:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVAFILE:file}:%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yy/MM/dd HH:mm:ss" + } + }, + "level":{ + "map_fieldvalue":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + } + } + } + ] +} + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml new file mode 100644 index 0000000..a080a20 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/configuration/storm-logsearch-conf.xml @@ -0,0 +1,110 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + Zookeeper + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + NIMBUS:storm_nimbus;SUPERVISOR:storm_supervisor,storm_worker,storm_logviewer;STORM_UI_SERVER:storm_ui;DRPC_SERVER:storm_drpc + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"storm_drpc", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/drpc.log" + }, + { + "type":"storm_logviewer", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/logviewer.log" + }, + { + "type":"storm_nimbus", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/nimbus.log" + }, + { + "type":"storm_supervisor", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/supervisor.log" + }, + { + "type":"storm_ui", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/ui.log" + }, + { + "type":"storm_worker", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/*worker*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "storm_drpc", + "storm_logviewer", + "storm_nimbus", + "storm_supervisor", + "storm_ui", + "storm_worker" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + } + } + } + ] + } + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml new file mode 100644 index 0000000..3c0abbf --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration-mapred/mapred-logsearch-conf.xml @@ -0,0 +1,80 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + MapReduce + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + HISTORYSERVER:mapred_historyserver + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"mapred_historyserver", + "rowtype":"service", + "path":"{{default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/mapred-env/mapred_user', 'mapred')}}/mapred-{{default('configurations/mapred-env/mapred_user', 'mapred')}}-historyserver*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "mapred_historyserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\$%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\$\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml new file mode 100644 index 0000000..95cf0c9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/configuration/yarn-logsearch-conf.xml @@ -0,0 +1,104 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + YARN + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + RESOURCEMANAGER:yarn_resourcemanager,yarn_historyserver,yarn_jobsummary;NODEMANAGER:yarn_nodemanager;APP_TIMELINE_SERVER:yarn_timelineserver + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"yarn_nodemanager", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-nodemanager-*.log" + }, + { + "type":"yarn_resourcemanager", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-resourcemanager-*.log" + }, + { + "type":"yarn_timelineserver", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-timelineserver-*.log" + }, + { + "type":"yarn_historyserver", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/yarn-{{default('configurations/yarn-env/yarn_user', 'yarn')}}-historyserver-*.log" + }, + { + "type":"yarn_jobsummary", + "rowtype":"service", + "path":"{{default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/yarn-env/yarn_user', 'yarn')}}/hadoop-mapreduce.jobsummary.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "yarn_historyserver", + "yarn_jobsummary", + "yarn_nodemanager", + "yarn_resourcemanager", + "yarn_timelineserver" + ] + } + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\$%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\$\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml new file mode 100644 index 0000000..0b8ab7a --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-logsearch-conf.xml @@ -0,0 +1,80 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + Zeppelin + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + ZEPPELIN_MASTER:zeppelin + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"zeppelin", + "rowtype":"service", + "path":"{{default('/configurations/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')}}/zeppelin-zeppelin-*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "zeppelin" + ] + } + }, + "log4j_format":"", + "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])", + "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\$\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\$%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] + } + + + content + false + + + + http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml new file mode 100644 index 0000000..325af14 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/ZOOKEEPER/3.4.5/configuration/zookeeper-logsearch-conf.xml @@ -0,0 +1,76 @@ + + + + + + service_name + Service name + Service name for Logsearch Portal (label) + Zookeeper + + + + component_mappings + Component mapping + Logsearch component logid mapping list (e.g.: COMPONENT1:logid1,logid2;COMPONENT2:logid3) + ZOOKEEPER_SERVER:zookeeper + + + + content + Logfeeder Config + Metadata jinja template for Logfeeder which contains grok patterns for reading service specific logs. + +{ + "input":[ + { + "type":"zookeeper", + "rowtype":"service", + "path":"{{default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')}}/zookeeper*.log" + } + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{"type":["zookeeper"]} + }, + "log4j_format":"%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + } + } + } + ] +} + + + content + false + + + + \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py index ba070fe..7a13637 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py @@ -62,7 +62,7 @@ class TestLogFeeder(RMFTestCase): ) self.assertResourceCalled('PropertiesFile', '/etc/ambari-logsearch-logfeeder/conf/logfeeder.properties', properties={'logfeeder.checkpoint.folder': '/etc/ambari-logsearch-logfeeder/conf/checkpoints', - 'logfeeder.config.files': 'global.config.json,output.config.json,input.config-accumulo.json,input.config-ambari.json,input.config-ams.json,input.config-atlas.json,input.config-falcon.json,input.config-flume.json,input.config-hbase.json,input.config-hdfs.json,input.config-hive.json,input.config-hst.json,input.config-infra.json,input.config-kafka.json,input.config-knox.json,input.config-logsearch.json,input.config-nifi.json,input.config-oozie.json,input.config-ranger.json,input.config-spark.json,input.config-spark2.json,input.config-storm.json,input.config-yarn.json,input.config-zeppelin.json,input.config-zookeeper.json', + 'logfeeder.config.files': 'global.config.json,output.config.json,input.config-ambari.json,input.config-logsearch.json,input.config-zookeeper.json', 'logfeeder.metrics.collector.hosts': '', 'logfeeder.solr.core.config.name': 'history', 'logfeeder.solr.zk_connect_string': 'c6401.ambari.apache.org:2181/infra-solr' @@ -80,9 +80,7 @@ class TestLogFeeder(RMFTestCase): encoding='utf-8' ) - logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume', 'hbase', 'hdfs', 'hive', 'hst', - 'infra', 'kafka', 'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'spark', 'spark2', 'storm', - 'yarn', 'zeppelin', 'zookeeper'] + logfeeder_supported_services = ['ambari','logsearch'] logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + \ ['input.config-%s.json' % (tag) for tag in logfeeder_supported_services] @@ -91,6 +89,9 @@ class TestLogFeeder(RMFTestCase): self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name, content=Template(file_name + ".j2") ) + self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-zookeeper.json', + content=InlineTemplate("pattern content") + ) def test_configure_default(self): self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/logfeeder.py", http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py index 8c0ed16..6b4fab8 100644 --- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py +++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logsearch.py @@ -173,7 +173,6 @@ class TestLogSearch(RMFTestCase): stack_version = self.STACK_VERSION, target = RMFTestCase.TARGET_COMMON_SERVICES ) - self.configureResourcesCalled() self.assertNoMoreResources() http://git-wip-us.apache.org/repos/asf/ambari/blob/d5e152af/ambari-server/src/test/python/stacks/2.4/configs/default.json ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.4/configs/default.json b/ambari-server/src/test/python/stacks/2.4/configs/default.json index a2c4791..c3eba53 100644 --- a/ambari-server/src/test/python/stacks/2.4/configs/default.json +++ b/ambari-server/src/test/python/stacks/2.4/configs/default.json @@ -229,6 +229,11 @@ "zoo.cfg": { "clientPort": "2181" }, + "zookeeper-logsearch-conf" : { + "service_name": "Zookeeper", + "component_mappings" : "ZOOKEEPER_SERVER:zookeeper", + "content" : "pattern content" + }, "logsearch-admin-json" : { "logsearch_admin_username" : "admin", "logsearch_admin_password" : "admin", @@ -314,9 +319,6 @@ }, "logfeeder-log4j": { "content": "<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n\n<!DOCTYPE log4j:configuration SYSTEM \"log4j.dtd\">\n<log4j:configuration xmlns:log4j=\"http://jakarta. apache.org/log4j/\">\n <appender name=\"console\" class=\"org.apache.log4j.ConsoleAppender\">\n <param name=\"Target\" value=\"System.out\" />\n <layout class=\"org.apache.log4j.PatternLayout\">\n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\" />\n </layout>\n </appender>\n\n <appender name=\"rolling_file\" class=\"org.apache.log4j.RollingFileAppender\"> \n <param name=\"file\" value=\"{{logfeeder_log_dir}}/logfeeder.log\" />\n <param name=\"append\" value=\"true\" /> \n <param name=\"maxFileSize\" value=\"10MB\" /> \n <param name=\"maxBackupIndex\" value=\"10\" /> \n <layout class=\"org.apache.log4j.PatternLayout\"> \n <param name=\"ConversionPattern\" value=\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\"/> \n </layout> \n </appender> \n\n <category name=\"org.apache.ambari.logfeeder\" additivity=\"false\">\n <priori ty value=\"info\" />\n <appender-ref ref=\"rolling_file\" />\n </category>\n\n <root>\n <priority value=\"warn\" />\n <appender-ref ref=\"rolling_file\" />\n </root>\n</log4j:configuration>" - }, - "logfeeder-input-configs": { - "content": "{\n \"global\":{\n \"add_fields\":{\n \"cluster\":\"{{cluster_name}}\"\n },\n \"source\":\"file\",\n \"tail\":\"true\",\n \"gen_event_md5\":\"true\",\n \"start_position\":\"beginning\"\n },\n \"input\":[\n {\n \"type\":\"accumulo_gc\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/gc_*.log\"\n },\n {\n \"type\":\"accumulo_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/master_*.log\"\n },\n {\n \"type\":\"accumulo_monitor\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/monitor_*.log\"\n },\n {\n \"type\":\"accumulo_tracer\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/tracer_*.log\"\n },\n {\n \"type\":\"accumulo_tserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{accumulo_log_dir}}/tserver_*.log\"\n },\n {\n \"type\":\"atlas_app\",\n \"rowtype\":\"se rvice\",\n \"path\":\"{{atlas_log_dir}}/application.log\"\n },\n {\n \"type\":\"ambari_agent\",\n \"rowtype\":\"service\",\n \"path\":\"{{ambari_agent_log_dir}}/ambari-agent.log\"\n },\n {\n \"type\":\"ambari_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{ambari_server_log_dir}}/ambari-server.log\"\n },\n {\n \"type\":\"ams_hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/hbase-ams-master-*.log\"\n },\n {\n \"type\":\"ams_hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/hbase-ams-regionserver-*.log\"\n },\n {\n \"type\":\"ams_collector\",\n \"rowtype\":\"service\",\n \"path\":\"{{metrics_collector_log_dir}}/ambari-metrics-collector.log\"\n },\n {\n \"type\":\"falcon_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{falcon_log_dir}}/falcon.application.log\"\n },\n {\ n \"type\":\"hbase_master\",\n \"rowtype\":\"service\",\n \"path\":\"{{hbase_log_dir}}/hbase-hbase-master-*.log\"\n },\n {\n \"type\":\"hbase_regionserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log\"\n },\n {\n \"type\":\"hdfs_datanode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-datanode-*.log\"\n },\n {\n \"type\":\"hdfs_namenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-namenode-*.log\"\n },\n {\n \"type\":\"hdfs_journalnode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-journalnode-*.log\"\n },\n {\n \"type\":\"hdfs_secondarynamenode\",\n \"rowtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-secondarynamenode-*.log\"\n },\n {\n \"type\":\"hdfs_zkfc\",\n \"r owtype\":\"service\",\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hadoop-hdfs-zkfc-*.log\"\n },\n {\n \"type\":\"hive_hiveserver2\",\n \"rowtype\":\"service\",\n \"path\":\"{{hive_log_dir}}/hiveserver2.log\"\n },\n {\n \"type\":\"hive_metastore\",\n \"rowtype\":\"service\",\n \"path\":\"{{hive_log_dir}}/hivemetastore.log\"\n },\n {\n \"type\":\"kafka_controller\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/controller.log\"\n },\n {\n \"type\":\"kafka_request\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/kafka-request.log\"\n },\n {\n \"type\":\"kafka_logcleaner\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/log-cleaner.log\"\n },\n {\n \"type\":\"kafka_server\",\n \"rowtype\":\"service\",\n \"path\":\"{{kafka_log_dir}}/server.log\"\n },\n {\n \"type\":\"kafka_statechange\",\n \"rowtype\":\"service \",\n \"path\":\"{{kafka_log_dir}}/state-change.log\"\n },\n {\n \"type\":\"knox_gateway\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/gateway.log\"\n },\n {\n \"type\":\"knox_cli\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/knoxcli.log\"\n },\n {\n \"type\":\"knox_ldap\",\n \"rowtype\":\"service\",\n \"path\":\"{{knox_log_dir}}/ldap.log\"\n },\n {\n \"type\":\"mapred_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{mapred_log_dir_prefix}}/mapred/mapred-mapred-historyserver*.log\"\n },\n {\n \"type\":\"logsearch_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{logsearch_log_dir}}/logsearch.log\"\n },\n {\n \"type\":\"logsearch_feeder\",\n \"rowtype\":\"service\",\n \"path\":\"{{logfeeder_log_dir}}/logfeeder.log\"\n },\n {\n \"type\":\"logsearch_perf\",\n \"rowtype\":\"service\",\n \"path\":\"{{l ogsearch_log_dir}}/logsearch-performance.log\"\n },\n {\n \"type\":\"ranger_admin\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_admin_log_dir}}/xa_portal.log\"\n },\n {\n \"type\":\"ranger_dbpatch\",\n \"is_enabled\":\"true\",\n \"path\":\"{{ranger_admin_log_dir}}/ranger_db_patch.log\"\n },\n {\n \"type\":\"ranger_kms\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_kms_log_dir}}/kms.log\"\n },\n {\n \"type\":\"ranger_usersync\",\n \"rowtype\":\"service\",\n \"path\":\"{{ranger_usersync_log_dir}}/usersync.log\"\n },\n {\n \"type\":\"oozie_app\",\n \"rowtype\":\"service\",\n \"path\":\"{{oozie_log_dir}}/oozie.log\"\n },\n {\n \"type\":\"yarn_nodemanager\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-nodemanager-*.log\"\n },\n {\n \"type\":\"yarn_resourcemanager\",\n \"rowtype\":\"service\",\n \"p ath\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-resourcemanager-*.log\"\n },\n {\n \"type\":\"yarn_timelineserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-timelineserver-*.log\"\n },\n {\n \"type\":\"yarn_historyserver\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/yarn-yarn-historyserver-*.log\"\n },\n {\n \"type\":\"yarn_jobsummary\",\n \"rowtype\":\"service\",\n \"path\":\"{{yarn_log_dir_prefix}}/yarn/hadoop-mapreduce.jobsummary.log\"\n },\n {\n \"type\":\"storm_drpc\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/drpc.log\"\n },\n {\n \"type\":\"storm_logviewer\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/logviewer.log\"\n },\n {\n \"type\":\"storm_nimbus\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/nimbus.log\"\n },\n {\n \"type\":\"st orm_supervisor\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/supervisor.log\"\n },\n {\n \"type\":\"storm_ui\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/ui.log\"\n },\n {\n \"type\":\"storm_worker\",\n \"rowtype\":\"service\",\n \"path\":\"{{storm_log_dir}}/*worker*.log\"\n },\n {\n \"type\":\"zookeeper\",\n \"rowtype\":\"service\",\n \"path\":\"{{zk_log_dir}}/zookeeper/zookeeper*.out\"\n },\n {\n \"type\":\"hdfs_audit\",\n \"rowtype\":\"audit\",\n \"is_enabled\":\"true\",\n \"add_fields\":{\n \"logType\":\"HDFSAudit\",\n \"enforcer\":\"hadoop-acl\",\n \"repoType\":\"1\",\n \"repo\":\"hdfs\"\n },\n \"path\":\"{{hdfs_log_dir_prefix}}/hdfs/hdfs-audit.log\"\n }\n \n ],\n \"filter\":[\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"accumulo_mas ter\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} [%-8c{2}] %-5p: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"comment\":\"This one has one extra space after LEVEL\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"accumulo_gc\",\n \"accumulo_monitor\",\n \"accumulo_tracer\",\n \"accumulo_tserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} [%-8c{2}] %-5p: %X{application} %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{JAVACLASS:logger_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"atlas_app\",\n \"falcon_app\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d %-5p - [%t:%x] ~ %m (%c{1}:%L)%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{SPACE}-%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}~%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\" :{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_collector\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %p %c: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ams_hbase_master\",\n \"ams_hbase_regionserver\",\n \"hbase_master\",\n \"hbase_regionserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t] %c{2}: %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ambari_agent\"\n ]\n \n }\n \n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})\",\n \" message_pattern\":\"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n },\n \"level\":{\n \"map_fieldvalue\":{\n \"pre_value\":\"WARNING\",\n \"post_value\":\"WARN\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ambari_server\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{DATE} %5p [%t] %c{1}:%L - %m%n\",\n \"multiline_pattern\":\"^(%{USER_SYNC_DATE:logtime})\",\n \"message_pattern\":\"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number} %{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"dd MMM yyyy HH:mm:ss\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_datanode\",\n \"hdfs_journalnode\",\n \"hdfs_secondarynamenode\",\n \"hdfs_namenode\",\n \"hdfs_zkfc\",\n \"knox_gateway\",\n \"knox_cli\",\n \"knox_ldap\",\n \"mapred_historyserver\",\n \"yarn_historyserver\",\n \"yarn_jobsummary\",\n \"yarn_nodemanager\",\n \"yarn_resourcemanager\",\n \"yarn_timelineserver\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTA MP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\$%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\$\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hive_hiveserver2\",\n \"hive_metastore\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p [%t]: %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]:%{SPACE}%{JAVACLASS:logger_name}%{SP ACE}\$%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\$\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_controller\",\n \"kafka_request\",\n \"kafka_logcleaner\"\n ]\n \n }\n \n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])\",\n \"message_pattern\":\"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,S SS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"comment\":\"Suppose to be same log4j pattern as other kafka processes, but some reason thread is not printed\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"kafka_server\",\n \"kafka_statechange\"\n ]\n \n }\n \n },\n \"log4j_format\":\"[%d] %p %m (%c)%n\",\n \"multiline_pattern\":\"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])\",\n \"message_pattern\":\"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"o ozie_app\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %5p %c{1}:%L - SERVER[${oozie.instance.id}] %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{DATA:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"logsearch_app\",\n \"logsearch_feeder\",\n \"logsearch_perf\",\n \"ranger_admin\",\n \"ranger_dbpatch\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d [%t] %-5p %C{6} (%F:%L) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\$%{JAVAFILE:file}:%{INT:line_number}\$%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ranger_kms\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{1} - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"ranger_usersync\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n\",\n \"multiline_pattern\":\"^(%{USER_SYNC_DATE:logtime})\",\n \"message_pattern\":\"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"dd MMM yyyy HH:mm:ss\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"storm_drpc\",\n \"storm_logviewer\",\n \"storm_nimbus\",\n \"storm_supervisor\",\n \"storm_ui\",\n \"storm_worker\"\n ]\n \n }\n \n },\n \"log4j_format\":\"\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss.SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"zookeeper\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} - %-5p [%t:%C{1}@% L] - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:logtime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}-%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\@%{INT:line_number}\\]%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"logtime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"log4j_format\":\"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n\",\n \"multiline_pattern\":\"^(%{TIMESTAMP_ISO8601:evtTime})\",\n \"message_pattern\":\"(?m)^%{TIMESTAMP_ISO8601:evtTime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}\",\n \"post_map_values\":{\n \"evtTime\":{\n \"map_date\":{\n \"date_pattern\":\"yyyy-MM-dd HH:mm:ss,SSS\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"keyvalue\",\n \"sort_order\":1,\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"source_field\":\"log_message\",\n \"value_split\":\"=\",\n \"field_split\":\"\t\",\n \"post_map_values\":{\n \"src\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"resource\"\n }\n \n },\n \"ip\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliIP\"\n }\n \n },\n \"allowed\":[\n {\n \"map_fieldvalue\":{\n \"pre_value\":\"true\",\n \"post_value\":\"1\"\n }\n \n },\n {\n \"map _fieldvalue\":{\n \"pre_value\":\"false\",\n \"post_value\":\"0\"\n }\n \n },\n {\n \"map_fieldname\":{\n \"new_fieldname\":\"result\"\n }\n \n }\n \n ],\n \"cmd\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"action\"\n }\n \n },\n \"proto\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"cliType\"\n }\n \n },\n \"callerContext\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"req_caller_id\"\n }\n \n }\n \n }\n \n },\n {\n \"filter\":\"grok\",\n \"sort_order\":2,\n \"source_field\":\"ugi\",\n \"remove_source_field\":\"false\",\n \"conditions\":{\n \"fields\":{\n \"type\":[\n \"hdfs_audit\"\n ]\n \n }\n \n },\n \"message_pattern\":\"%{USERNAME:p_user}.+auth:%{USERNAME:p_authType}.+via %{USERNAME:k_user}.+auth:%{USERNAME:k_authType}|%{USERNAME:user}.+auth:%{USERNAME:authType}|%{USERNAME:x_user}\",\n \"post_map_values\":{\n \"user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"x_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"p_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"reqUser\"\n }\n \n },\n \"k_user\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"proxyUsers\"\n }\n \n },\n \"p_authType\":{\n \"map_fieldname\":{\n \"new_fieldname\":\"authType\"\n }\n \n },\n \"k_authType\":{\n \"map_fieldname\":{\n \"new_fie ldname\":\"proxyAuthType\"\n }\n \n }\n \n }\n \n }\n \n ],\n \"output\":[\n {\n \"is_enabled\":\"{{solr_service_logs_enable}}\",\n \"comment\":\"Output to solr for service logs\",\n \"destination\":\"solr\",\n \"zk_connect_string\":\"{{zookeeper_quorum}}{{solr_znode}}\",\n \"collection\":\"{{solr_collection_service_logs}}\",\n \"number_of_shards\": \"{{logsearch_collection_service_logs_numshards}}\",\n \"splits_interval_mins\": \"{{logsearch_service_logs_split_interval_mins}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"service\"\n ]\n \n }\n \n }\n \n },\n {\n \"comment\":\"Output to solr for audit records\",\n \"is_enabled\":\"{{solr_audit_logs_enable}}\",\n \"destination\":\"solr\",\n \"zk_connect_string\":\"{{zookeeper_quorum}}{{solr_znode}}\",\n \"collection\":\"{{solr_collec tion_audit_logs}}\",\n \"number_of_shards\": \"{{logsearch_collection_audit_logs_numshards}}\",\n \"splits_interval_mins\": \"{{logsearch_audit_logs_split_interval_mins}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"audit\"\n ]\n \n }\n \n }\n \n },\n {\n \"is_enabled\":\"{{kafka_service_logs_enable}}\",\n \"destination\":\"kafka\",\n \"broker_list\":\"{{kafka_broker_list}}\",\n \"topic\":\"{{kafka_topic_service_logs}}\",\n \"kafka.security.protocol\":\"{{kafka_security_protocol}}\",\n \"kafka.sasl.kerberos.service.name\":\"{{kafka_kerberos_service_name}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"service\"\n ]\n \n }\n \n }\n \n },\n {\n \"is_enabled\":\"{{kafka_audit_logs_enable}}\",\n \"destination\":\"kafka\",\n \"broker_list\":\"{{kafka_brok er_list}}\",\n \"topic\":\"{{kafka_topic_audit_logs}}\",\n \"kafka.security.protocol\":\"{{kafka_security_protocol}}\",\n \"kafka.sasl.kerberos.service.name\":\"{{kafka_kerberos_service_name}}\",\n \"conditions\":{\n \"fields\":{\n \"rowtype\":[\n \"audit\"\n ]\n \n }\n \n }\n \n }\n \n ]\n \n}" } }, "configuration_attributes": {