ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From oleew...@apache.org
Subject [2/2] ambari git commit: AMBARI-17424. Add missing components to LogSearch Part 2 (Miklos Gergely via oleewere)
Date Fri, 24 Jun 2016 21:03:45 GMT
AMBARI-17424. Add missing components to LogSearch Part 2 (Miklos Gergely via oleewere)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4207e601
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4207e601
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4207e601

Branch: refs/heads/trunk
Commit: 4207e60186c42215ead3283285b814af9fe37067
Parents: 36d1f86
Author: Miklos Gergely <mgergely@hortonworks.com>
Authored: Fri Jun 24 17:18:18 2016 +0200
Committer: oleewere <oleewere@gmail.com>
Committed: Fri Jun 24 23:00:12 2016 +0200

----------------------------------------------------------------------
 .../src/main/resources/HadoopServiceConfig.json | 29 ++++++++++
 .../FLUME/1.4.0.2.0/metainfo.xml                |  6 +++
 .../LOGSEARCH/0.5.0/metainfo.xml                |  6 +++
 .../LOGSEARCH/0.5.0/package/scripts/params.py   |  6 ++-
 .../templates/input.config-flume.json.j2        | 56 ++++++++++++++++++++
 .../templates/input.config-hbase.json.j2        | 29 ++++++++++
 .../templates/input.config-logsearch.json.j2    | 30 +++++++++++
 .../templates/input.config-zeppelin.json.j2     | 56 ++++++++++++++++++++
 .../ZEPPELIN/0.6.0.2.5/metainfo.xml             |  6 +++
 .../stacks/HDP/2.3/services/HBASE/metainfo.xml  |  6 +++
 .../stacks/2.4/LOGSEARCH/test_logfeeder.py      |  4 +-
 11 files changed, 230 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
----------------------------------------------------------------------
diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
index 73400e3..abe692d 100644
--- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
+++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/HadoopServiceConfig.json
@@ -100,6 +100,18 @@
       ]
       
     },
+    "flume":{
+      "label":"Flume",
+      "components":[
+        {
+          "name":"flume_handler"
+        }
+      ],
+      "dependencies":[
+        
+      ]
+      
+    },
     "hbase":{
       "label":"HBase",
       "components":[
@@ -108,6 +120,9 @@
         },
         {
           "name":"hbase_regionserver"
+        },
+        {
+          "name":"hbase_phoenix_server"
         }
         
       ],
@@ -272,6 +287,9 @@
         },
         {
           "name":"logsearch_perf"
+        },
+        {
+          "name":"logsearch_solr"
         }
         
       ],
@@ -425,6 +443,17 @@
       ]
       
     },
+    "zeppelin":{
+      "label":"Zeppelin",
+      "components":[
+        {
+          "name":"zeppelin"
+        }
+      ],
+      "dependencies":[
+        
+      ]
+    },
     "zookeeper":{
       "label":"ZooKeeper",
       "components":[

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
index 0396c28..b4f1058 100644
--- a/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/FLUME/1.4.0.2.0/metainfo.xml
@@ -35,6 +35,12 @@
             <scriptType>PYTHON</scriptType>
             <timeout>1200</timeout>
           </commandScript>
+          <logs>
+            <log>
+              <logId>flume_handler</logId>
+              <primary>true</primary>
+            </log>
+          </logs>
         </component>
       </components>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
index e3a8f98..ffa9e13 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/metainfo.xml
@@ -84,6 +84,12 @@
             <script>scripts/logsearch_solr.py</script>
             <scriptType>PYTHON</scriptType>
           </commandScript>
+          <logs>
+            <log>
+              <logId>logsearch_solr</logId>
+              <primary>true</primary>
+            </log>
+          </logs>
           <dependencies>
             <dependency>
               <name>LOGSEARCH/LOGSEARCH_SOLR_CLIENT</name>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index d21a97f..4bc1004 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -247,6 +247,7 @@ metrics_monitor_log_dir = default('/configurations/ams-env/metrics_monitor_log_d
 atlas_log_dir = default('/configurations/atlas-env/metadata_log_dir', '/var/log/atlas')
 accumulo_log_dir = default('/configurations/accumulo-env/accumulo_log_dir', '/var/log/accumulo')
 falcon_log_dir = default('/configurations/falcon-env/falcon_log_dir', '/var/log/falcon')
+flume_log_dir = default('/configurations/flume-env/flume_log_dir', '/var/log/flume')
 hbase_log_dir = default('/configurations/hbase-env/hbase_log_dir', '/var/log/hbase')
 hdfs_log_dir_prefix = default('/configurations/hadoop-env/hdfs_log_dir_prefix', '/var/log/hadoop')
 hive_log_dir = default('/configurations/hive-env/hive_log_dir', '/var/log/hive')
@@ -259,6 +260,7 @@ ranger_kms_log_dir = default('/configurations/kms-env/kms_log_dir', '/var/log/ra
 storm_log_dir = default('/configurations/storm-env/storm_log_dir', '/var/log/storm')
 yarn_log_dir_prefix = default('/configurations/yarn-env/yarn_log_dir_prefix', '/var/log/hadoop')
 mapred_log_dir_prefix = default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')
+zeppelin_log_dir = default('/configuration/zeppelin-env/zeppelin_log_dir', '/var/log/zeppelin')
 zk_log_dir = default('/configurations/zookeeper-env/zk_log_dir', '/var/log/zookeeper')
 
 #####################################
@@ -328,8 +330,8 @@ logfeeder_checkpoint_folder = default('/configurations/logfeeder-env/logfeeder.c
 logfeeder_log_filter_enable = str(default('/configurations/logfeeder-properties/logfeeder.log.filter.enable',
True)).lower()
 logfeeder_solr_config_interval = default('/configurations/logfeeder-properties/logfeeder.solr.config.interval',
5)
 
-logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase',
'hdfs', 'hive', 'kafka',
-                                'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm',
'yarn', 'zookeeper']
+logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume',
'hbase', 'hdfs', 'hive', 'kafka',
+                                'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm',
'yarn', 'zeppelin', 'zookeeper']
 
 logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json'
% (tag) for tag in
                                                                               logfeeder_supported_services]

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
new file mode 100644
index 0000000..d1ea632
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-flume.json.j2
@@ -0,0 +1,56 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #   http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+  "input":[
+    {
+      "type":"flume_handler",
+      "rowtype":"service",
+      "path":"{{flume_log_dir}}/flume.log"
+    }
+
+  ],
+  "filter":[
+    {
+      "filter":"grok",
+      "conditions":{
+        "fields":{
+          "type":[
+            "flume_handler"
+          ]
+
+        }
+
+      },
+      "log4j_format":"",
+      "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})",
+      "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}\\(%{JAVACLASS:class}\\.%{JAVAMETHOD:method}:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+      "post_map_values":{
+        "logtime":{
+          "map_date":{
+            "date_pattern":"dd MMM yyyy HH:mm:ss,SSS"
+          }
+
+        }
+
+      }
+
+    }
+
+  ]
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
index fb47e77..42c5683 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-hbase.json.j2
@@ -26,6 +26,11 @@
       "type":"hbase_regionserver",
       "rowtype":"service",
       "path":"{{hbase_log_dir}}/hbase-hbase-regionserver-*.log"
+    },
+    {
+      "type":"hbase_phoenix_server",
+      "rowtype":"service",
+      "path":"{{hbase_log_dir}}/phoenix-hbase-server.log"
     }
 
   ],
@@ -55,6 +60,30 @@
 
       }
 
+    },
+    {
+      "filter":"grok",
+      "conditions":{
+        "fields":{
+          "type":[
+            "hbase_phoenix_server"
+          ]
+
+        }
+      },
+      "log4j_format":"%d{ISO8601} %-5p [%t] %c{2}: %m%n",
+      "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+      "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}:%{SPACE}%{GREEDYDATA:log_message}",
+      "post_map_values":{
+        "logtime":{
+          "map_date":{
+            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+          }
+
+        }
+
+      }
+
     }
 
   ]

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
index ba151b9..31c37bf 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-logsearch.json.j2
@@ -31,6 +31,11 @@
       "type":"logsearch_perf",
       "rowtype":"service",
       "path":"{{logsearch_log_dir}}/logsearch-performance.json"
+    },
+    {
+      "type":"logsearch_solr",
+      "rowtype":"service",
+      "path":"{{logsearch_solr_log_dir}}/solr.log"
     }
 
   ],
@@ -49,6 +54,31 @@
 
       }
 
+    },
+    {
+      "filter":"grok",
+      "conditions":{
+        "fields":{
+          "type":[
+            "logsearch_solr"
+          ]
+
+        }
+
+      },
+      "log4j_format":"",
+      "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})",
+      "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}",
+      "post_map_values":{
+        "logtime":{
+          "map_date":{
+            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+          }
+
+        }
+
+      }
+
     }
 
   ]

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
new file mode 100644
index 0000000..09cd8b3
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/input.config-zeppelin.json.j2
@@ -0,0 +1,56 @@
+{#
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements.  See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership.  The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License.  You may obtain a copy of the License at
+ #
+ #   http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ #}
+{
+  "input":[
+    {
+      "type":"zeppelin",
+      "rowtype":"service",
+      "path":"{{zeppelin_log_dir}}/zeppelin-zeppelin-*.log"
+    }
+
+  ],
+  "filter":[
+    {
+      "filter":"grok",
+      "conditions":{
+        "fields":{
+          "type":[
+            "zeppelin"
+          ]
+
+        }
+
+      },
+      "log4j_format":"",
+      "multiline_pattern":"^(%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\])",
+      "message_pattern":"(?m)^%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}\\(\\{{"{"}}%{DATA:thread_name}\\{{"}"}}%{SPACE}%{JAVAFILE:file}\\[%{JAVAMETHOD:method}\\]:%{INT:line_number}\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}",
+      "post_map_values":{
+        "logtime":{
+          "map_date":{
+            "date_pattern":"yyyy-MM-dd HH:mm:ss,SSS"
+          }
+
+        }
+
+      }
+
+    }
+
+  ]
+
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
index c40a593..4f19b43 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
@@ -53,6 +53,12 @@ limitations under the License.
               </auto-deploy>
             </dependency>
           </dependencies>
+          <logs>
+            <log>
+              <logId>zeppelin</logId>
+              <primary>true</primary>
+            </log>
+          </logs>
         </component>
       </components>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
index 0614cf7..d7401d8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HBASE/metainfo.xml
@@ -33,6 +33,12 @@
             <script>scripts/phoenix_queryserver.py</script>
             <scriptType>PYTHON</scriptType>
           </commandScript>
+          <logs>
+            <log>
+              <logId>hbase_phoenix_server</logId>
+              <primary>true</primary>
+            </log>
+          </logs>
         </component>
       </components>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4207e601/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 0613770..26e8934 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -98,8 +98,8 @@ class TestLogFeeder(RMFTestCase):
                               encoding='utf-8'
                               )
 
-    logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'hbase',
'hdfs', 'hive', 'kafka',
-                                    'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm',
'yarn', 'zookeeper']
+    logfeeder_supported_services = ['accumulo', 'ambari', 'ams', 'atlas', 'falcon', 'flume',
'hbase', 'hdfs', 'hive', 'kafka',
+                                    'knox', 'logsearch', 'nifi', 'oozie', 'ranger', 'storm',
'yarn', 'zeppelin', 'zookeeper']
 
     logfeeder_config_file_names = ['global.config.json', 'output.config.json'] + ['input.config-%s.json'
% (tag) for tag
                                                                                   in logfeeder_supported_services]


Mime
View raw message