ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From srima...@apache.org
Subject ambari git commit: AMBARI-16451. Takeover script configs results in conflict message that look wrong (aonishuk via srimanth)
Date Wed, 11 May 2016 16:44:21 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 41aa24740 -> a23fbd763


AMBARI-16451. Takeover script configs results in conflict message that look wrong (aonishuk
via srimanth)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a23fbd76
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a23fbd76
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a23fbd76

Branch: refs/heads/trunk
Commit: a23fbd7639373fd1f2bfb5d643871a094e302329
Parents: 41aa247
Author: Srimanth Gunturi <sgunturi@hortonworks.com>
Authored: Wed May 11 09:43:32 2016 -0700
Committer: Srimanth Gunturi <sgunturi@hortonworks.com>
Committed: Wed May 11 09:43:32 2016 -0700

----------------------------------------------------------------------
 .../resources/scripts/takeover_config_merge.py  | 79 +++++++++++++++++---
 1 file changed, 70 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a23fbd76/ambari-server/src/main/resources/scripts/takeover_config_merge.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/takeover_config_merge.py b/ambari-server/src/main/resources/scripts/takeover_config_merge.py
index 1432e4b..f940f27 100644
--- a/ambari-server/src/main/resources/scripts/takeover_config_merge.py
+++ b/ambari-server/src/main/resources/scripts/takeover_config_merge.py
@@ -28,7 +28,6 @@ import base64
 import time
 import xml
 import xml.etree.ElementTree as ET
-import yaml
 import StringIO
 import ConfigParser
 
@@ -42,20 +41,54 @@ Example:
 "c6401.ambari.apache.org/etc/hive/conf/log4j.properties" : "hive-log4j"}
 """
 
+LICENSE = """
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+
+
 class Parser:
   pass
 
+class ShParser(Parser):
+  def read_data_to_map(self, path):
+    with open(path, 'r') as file:
+      file_content = file.read()
+    return {"content" : file_content}, None
+
 class YamlParser(Parser): # Used Yaml parser to read data into a map
   def read_data_to_map(self, path):
+    try:
+      import yaml
+    except ImportError:
+      logger.error("Module PyYAML not installed. Please try to execute \"pip install pyyaml\"
for installing PyYAML module.")
+      sys.exit(1)
+
     configurations = {}
     with open(path, 'r') as file:
       try:
         for name, value in yaml.load(file).iteritems():
           if name != None:
             configurations[name] = str(value)
-      except yaml.YAMLError:
-        logger.exception("Yaml parser error: ")
-        return None
+      except:
+        logger.error("Couldn't parse {0} file. Skipping ...".format(path))
+        return None, None
     return configurations, None
 
 
@@ -121,13 +154,20 @@ class ConfigMerge:
   OUTPUT_DIR = '/tmp'
   OUT_FILENAME = 'ambari_takeover_config_merge.out'
   JSON_FILENAME = 'ambari_takeover_config_merge.json'
-  PARSER_BY_EXTENSIONS = {'.xml' : XmlParser(), '.yaml' : YamlParser(), '.properties' : PropertiesParser()}
-  SUPPORTED_EXTENSIONS = ['.xml', '.yaml', '.properties']
+  PARSER_BY_EXTENSIONS = {'.xml' : XmlParser(), '.yaml' : YamlParser(), '.properties' : PropertiesParser(),
'.sh' : ShParser()}
+  SUPPORTED_EXTENSIONS = ['.xml', '.yaml', '.properties', '.sh']
+  SUPPORTED_FILENAME_ENDINGS = {".sh" : "-env"}
   UNKNOWN_FILES_MAPPING_FILE = None
   SERVICE_TO_AMBARI_CONFIG_NAME = {
-    "storm.yaml": "storm-site"
+    "storm.yaml": "storm-site",
+    "runtime.properties" : "falcon-runtime",
+    "startup.properties" : "falcon-startup",
+    "flume-conf.properties" : "flume-conf",
+    "pig.properties" : "pig-properties"
   }
 
+  CONFIGS_WITH_CONTENT = ['pig-properties', '-log4j']
+
   NOT_MAPPED_FILES = ['log4j.properties']
 
 
@@ -144,7 +184,9 @@ class ConfigMerge:
         root, ext = os.path.splitext(file)
         if ext in extensions:
           file_path = os.path.join(dirName, file)
-
+          if ext in ConfigMerge.SUPPORTED_FILENAME_ENDINGS and not ConfigMerge.SUPPORTED_FILENAME_ENDINGS[ext]
in root:
+            logger.warn("File {0} is not configurable by Ambari. Skipping...".format(file_path))
+            continue
           config_name = None
           if file in ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME:
             config_name = ConfigMerge.SERVICE_TO_AMBARI_CONFIG_NAME[file]
@@ -196,8 +238,22 @@ class ConfigMerge:
   def format_for_blueprint(configurations, attributes):
     all_configs = []
     for configuration_type, configuration_properties in configurations.iteritems():
+      is_content = False
       all_configs.append({})
-      all_configs[-1][configuration_type] = {'properties' :configuration_properties}
+
+      for config_with_content in ConfigMerge.CONFIGS_WITH_CONTENT:
+        if config_with_content in configuration_type:
+          is_content = True
+          break
+
+      if is_content:
+        content = LICENSE
+        for property_name, property_value in configuration_properties.iteritems():
+          content+=property_name + "=" + property_value + "\n"
+        all_configs[-1][configuration_type] = {'properties': {"content" : content}}
+      else:
+        all_configs[-1][configuration_type] = {'properties' :configuration_properties}
+
       for configuration_type_attributes, properties_attributes in attributes.iteritems():
         if properties_attributes and configuration_type == configuration_type_attributes:
           all_configs[-1][configuration_type].update({"properties_attributes" : {"final"
: properties_attributes}})
@@ -215,7 +271,12 @@ class ConfigMerge:
       if len(value_to_filepaths) == 1:
         continue
 
+      first_item = False
       for value, filepaths in value_to_filepaths.iteritems():
+        if not first_item:
+          first_item = True
+          output += "\n\n=== {0} | {1} | {2} |\nHas conflicts with:\n\n".format(property_name,filepaths[0],
value)
+          continue
         for filepath in filepaths:
           output += "| {0} | {1} | {2} |\n".format(property_name, filepath, value)
 


Mime
View raw message