ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From swa...@apache.org
Subject [16/24] ambari git commit: AMBARI-18854. Add ability to add custom input descriptor to logfeeder (Miklos Gergely via oleewere)
Date Wed, 16 Nov 2016 21:58:49 GMT
AMBARI-18854. Add ability to add custom input descriptor to logfeeder (Miklos Gergely via oleewere)

Change-Id: I659c5fbc94af8ba54a26698e5b1ebe32fedeb874


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/50a36810
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/50a36810
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/50a36810

Branch: refs/heads/branch-feature-AMBARI-18901
Commit: 50a36810bf9bfafb50f13f11bfe39d6b88972b74
Parents: 7d2fa09
Author: Miklos Gergely <mgergely@hortonworks.com>
Authored: Wed Nov 16 11:47:39 2016 +0100
Committer: oleewere <oleewere@gmail.com>
Committed: Wed Nov 16 16:48:11 2016 +0100

----------------------------------------------------------------------
 .../logfeeder-custom-logsearch-conf.xml         | 46 ++++++++++++++++++++
 .../scripts/logsearch_config_aggregator.py      | 14 +++++-
 .../LOGSEARCH/0.5.0/package/scripts/params.py   |  1 +
 .../0.5.0/package/scripts/setup_logfeeder.py    |  1 +
 .../templates/HadoopServiceConfig.json.j2       | 14 +++---
 .../stacks/2.4/LOGSEARCH/test_logfeeder.py      |  3 ++
 6 files changed, 69 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/50a36810/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml
new file mode 100644
index 0000000..8358266
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/configuration/logfeeder-custom-logsearch-conf.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+  <property>
+    <name>service_component_mappings</name>
+    <display-name>Service Component mapping</display-name>
+    <description>Logsearch service component logid mapping list (e.g.: SERVICE1=S1_COMPONENT1:logid1,logid2;S1_COMPONENT2:logid3|SERVICE2=...)</description>
+    <value></value>
+    <value-attributes>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>content</name>
+    <display-name>Logfeeder Config</display-name>
+    <description>Metadata jinja template for Logfeeder which contains grok patterns
for reading service specific logs.</description>
+    <value>
+    </value>
+    <value-attributes>
+      <type>content</type>
+      <show-property-name>false</show-property-name>
+      <empty-value-valid>true</empty-value-valid>
+    </value-attributes>
+    <on-ambari-upgrade add="true"/>
+  </property>
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/50a36810/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py
index ad515ee..c14900e 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/logsearch_config_aggregator.py
@@ -46,7 +46,7 @@ def get_logfeeder_metadata(logsearch_meta_configs):
   """
   logfeeder_contents = {}
   for key, value in logsearch_meta_configs.iteritems():
-    if 'content' in logsearch_meta_configs[key]:
+    if 'content' in logsearch_meta_configs[key] and logsearch_meta_configs[key]['content'].strip():
       logfeeder_contents[key] = logsearch_meta_configs[key]['content']
       Logger.info("Found logfeeder pattern content in " + key)
   return logfeeder_contents
@@ -61,7 +61,17 @@ def get_logsearch_metadata(logsearch_meta_configs):
     if 'service_name' in logsearch_meta_configs[key] and 'component_mappings' in logsearch_meta_configs[key]:
       service_name = logsearch_meta_configs[key]['service_name']
       component_mappings = __parse_component_mappings(logsearch_meta_configs[key]['component_mappings'])
-      logsearch_service_component_mappings[service_name] = component_mappings
+      if service_name.strip() and component_mappings:
+        logsearch_service_component_mappings[service_name] = component_mappings
+    if 'service_component_mappings' in logsearch_meta_configs[key]:
+      service_component_mappings = logsearch_meta_configs[key]['service_component_mappings']
+      if service_component_mappings.strip():
+        for service_component_mapping in service_component_mappings.split('|'):
+          tokens = service_component_mapping.split('=')
+          service_name = tokens[0]
+          component_mappings = __parse_component_mappings(tokens[1])
+          if service_name.strip() and component_mappings:
+            logsearch_service_component_mappings[service_name] = component_mappings
 
   return logsearch_service_component_mappings
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/50a36810/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
index 1f5064a..bdb0c10 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/params.py
@@ -147,6 +147,7 @@ ambari_server_log_dir = '/var/log/ambari-server'
 ambari_agent_log_dir = '/var/log/ambari-agent'
 hst_log_dir = '/var/log/hst'
 hst_activity_log_dir = '/var/log/smartsense-activity'
+nifi_log_dir = default('/configurations/nifi-env/nifi_node_log_dir', '/var/log/nifi')
 
 # System logs
 logfeeder_system_messages_content = config['configurations']['logfeeder-system_log-env']['logfeeder_system_messages_content']

http://git-wip-us.apache.org/repos/asf/ambari/blob/50a36810/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
index e90d623..021c167 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/scripts/setup_logfeeder.py
@@ -67,6 +67,7 @@ def setup_logfeeder():
          content=Template(file_name + ".j2")
          )
 
+  File(format("{logsearch_logfeeder_conf}/input.config-logfeeder-custom.json"), action='delete')
   for service, pattern_content in params.logfeeder_metadata.iteritems():
     File(format("{logsearch_logfeeder_conf}/input.config-" + service.replace('-logsearch-conf',
'') + ".json"),
       content=InlineTemplate(pattern_content, extra_imports=[default])

http://git-wip-us.apache.org/repos/asf/ambari/blob/50a36810/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
index 7e9153f..efa51fa 100644
--- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
+++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/package/templates/HadoopServiceConfig.json.j2
@@ -18,24 +18,22 @@
  {
    "service": {
 {% if logsearch_metadata and logsearch_metadata.keys()|length > 0 %}
-  {% for metadata_key, component_list in logsearch_metadata.iteritems() %}
+{%   for metadata_key, component_list in logsearch_metadata.iteritems() %}
     "{{ metadata_key.lower() }}": {
       "label": "{{ metadata_key }}",
       "components": [
-    {% for component in component_list %}
+{%     for component in component_list %}
         {
           "name": "{{ component }}"
-        }
-      {% if not loop.last %}
-        ,
-      {% endif %}
-    {% endfor %}
+        }{% if not loop.last %},{% endif %}
+
+{%     endfor %}
       ],
       "dependencies": [
       ]
     }
      ,
-  {% endfor %}
+{%   endfor %}
 {% endif %}
 {% if logfeeder_system_log_enabled %}
      "system" : {

http://git-wip-us.apache.org/repos/asf/ambari/blob/50a36810/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
index 7a13637..773b75a 100644
--- a/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
+++ b/ambari-server/src/test/python/stacks/2.4/LOGSEARCH/test_logfeeder.py
@@ -89,6 +89,9 @@ class TestLogFeeder(RMFTestCase):
       self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/' + file_name,
                                 content=Template(file_name + ".j2")
                                 )
+    self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-logfeeder-custom.json',
+                              action=['delete']
+                              )
     self.assertResourceCalled('File', '/etc/ambari-logsearch-logfeeder/conf/input.config-zookeeper.json',
                               content=InlineTemplate("pattern content")
                               )


Mime
View raw message