ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vbrodets...@apache.org
Subject ambari git commit: AMBARI-12350. Storm manual upgrade from HDP 2.2 to HDP 2.3 leaves nimbus.seeds as a string value.(vbrodetskyi)
Date Thu, 09 Jul 2015 08:44:06 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 49c88a723 -> fa206f755


AMBARI-12350. Storm manual upgrade from HDP 2.2 to HDP 2.3 leaves nimbus.seeds as a string
value.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/fa206f75
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/fa206f75
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/fa206f75

Branch: refs/heads/branch-2.1
Commit: fa206f755f4bee0389ae65065c280ac5548b0a6b
Parents: 49c88a7
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Thu Jul 9 11:43:43 2015 +0300
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Thu Jul 9 11:43:43 2015 +0300

----------------------------------------------------------------------
 ambari-server/src/main/python/upgradeHelper.py  |  94 +++++++++++---
 .../catalog/UpgradeCatalog_2.1_to_2.3.json      |   5 +-
 .../catalog/UpgradeCatalog_2.2_to_2.3.json      |   5 +-
 .../src/test/python/TestUpgradeHelper.py        | 124 +++++++++++++++++++
 4 files changed, 211 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/fa206f75/ambari-server/src/main/python/upgradeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/upgradeHelper.py b/ambari-server/src/main/python/upgradeHelper.py
index dd72bdc..b2e0fa4 100644
--- a/ambari-server/src/main/python/upgradeHelper.py
+++ b/ambari-server/src/main/python/upgradeHelper.py
@@ -79,6 +79,11 @@ Example:
           "to-catalog": "test",          (optional, require "from-catalog. Target of new_property1_name)
           "default": "default value",    (optional, if set and old property not exists, new
one would be created with default value)
           "template": "yes",             (optional, template parsing for default option)
+          "coerce-to": "pre-defined type", (optional, convert value from one type to another.
Types supported:
+                                              yaml-array - converts string item1,item2 to
 ['item1', 'item2']
+                                            )
+          "replace-from": "something", (optional, should be present both from and to. Replace
'from' value to 'to')
+          "replace-to": "something,
           "required-services": ["YARN"]  (optional, process entry if services in the list
existed on the cluster
       }
      }
@@ -275,6 +280,10 @@ class CatConst(Const):
   PROPERTY_DEFAULT = "default"
   MERGED_COPY_TAG = "merged-copy"
   REQUIRED_SERVICES = "required-services"
+  COERCE_TO_PROPERTY_TAG = "coerce-to"
+  COERCE_YAML_OPTION_TAG = "yaml-array"
+  REPLACE_FROM_TAG = "replace-from"
+  REPLACE_TO_TAG = "replace-to"
   ITEMS_TAG = "items"
   TYPE_TAG = "type"
   TRUE_TAG = "yes"
@@ -385,6 +394,20 @@ class UpgradeCatalog(object):
     """
     for config_item in properties:
       cfg_item = properties[config_item]
+
+      """
+        case when "properties": {
+                        "yarn-site": {
+                            .....
+                         }
+                     }
+        is set like "properties": {
+           "yarn-site": ""
+        }
+      """
+      if not isinstance(cfg_item, dict):
+        raise MalformedPropertyDefinitionException("The property catalog '%s' definition
error" % config_item)
+
       properties[config_item] = dict(zip(
         cfg_item.keys(),
         map(lambda x: x if isinstance(x, dict) or isinstance(x, list) else {CatConst.PROPERTY_VALUE_TAG:
x}, cfg_item.values())
@@ -580,10 +603,9 @@ class PropertyMapping(object):
 
 
 class ServerConfigFactory(object):
-  _server_catalogs = {}
-
   def __init__(self):
     self.__observers = []
+    self._server_catalogs = {}
     self._load_configs()
 
   def subscribe(self, name, config_item):
@@ -653,6 +675,28 @@ class ServerConfigFactory(object):
       )
       item[CatConst.PROPERTY_DEFAULT] = parsed_value
 
+  def _process_property_value_transformation(self, catalog, property_map_definition, old_value):
+    """
+    :type catalog: UpgradeCatalog
+    :type property_map_definition: dict
+    :type old_value: str
+    :return: str
+    """
+
+    tmp = old_value
+
+    if CatConst.REPLACE_FROM_TAG in property_map_definition and CatConst.REPLACE_TO_TAG in
property_map_definition and\
+      property_map_definition[CatConst.REPLACE_TO_TAG] is not None and property_map_definition[CatConst.REPLACE_FROM_TAG]
is not None:
+      tmp = tmp.replace(property_map_definition[CatConst.REPLACE_FROM_TAG], property_map_definition[CatConst.REPLACE_TO_TAG])
+
+    if CatConst.COERCE_TO_PROPERTY_TAG in property_map_definition:
+      if property_map_definition[CatConst.COERCE_TO_PROPERTY_TAG] == CatConst.COERCE_YAML_OPTION_TAG:
+        # for example c6401,c6402 into ['c6401','c6402']
+        data = list(map(lambda x: "'%s'" % x.strip(), tmp.split(',')))
+        tmp = "[%s]" % ','.join(data)
+
+    return tmp
+
   def _process_single_map_transformation(self, catalog, map_item_name, map_property_item):
     """
     :type catalog UpgradeCatalog
@@ -670,6 +714,13 @@ class ServerConfigFactory(object):
     if CatConst.PROPERTY_MAP_TO in map_property_item:
       new_property_name = map_property_item[CatConst.PROPERTY_MAP_TO]
 
+    # process first required section
+    required_services = map_property_item[CatConst.REQUIRED_SERVICES] if CatConst.REQUIRED_SERVICES
in map_property_item else None
+
+    # process required-services tag
+    if required_services is not None and not is_services_exists(required_services):
+      return 0
+
     # process template tag
     self._process_default_template_map_replacement(catalog, map_property_item)
 
@@ -679,22 +730,22 @@ class ServerConfigFactory(object):
                                                                           map_property_item[CatConst.PROPERTY_TO_CATALOG]
!= ""else None
     default_value = map_property_item[CatConst.PROPERTY_DEFAULT] if CatConst.PROPERTY_DEFAULT
in map_property_item and \
                                                                     map_property_item[CatConst.PROPERTY_DEFAULT]
!= "" else None
-    required_services = map_property_item[CatConst.REQUIRED_SERVICES] if CatConst.REQUIRED_SERVICES
in map_property_item else None
-
-    # process required-services tag
-    if required_services is not None and not is_services_exists(required_services):
-      return 0
 
     if source_cfg_group is None and target_cfg_group is None:  # global scope mapping renaming
-      self.notify_observers(CatConst.ACTION_RENAME_PROPERTY, [old_property_name, new_property_name])
+      self.notify_observers(CatConst.ACTION_RENAME_PROPERTY, [old_property_name, new_property_name,
+                                                              self._process_property_value_transformation,
+                                                              catalog,
+                                                              map_property_item
+                                                              ])
     elif source_cfg_group is not None and target_cfg_group is not None:  # group-to-group
moving
       if source_cfg_group in self._server_catalogs and target_cfg_group in self._server_catalogs:
         old_cfg_group = self.get_config(source_cfg_group).properties
         new_cfg_group = self.get_config(target_cfg_group).properties
 
         if old_property_name in old_cfg_group:
-          new_cfg_group[new_property_name] = old_cfg_group[old_property_name]
-          del old_cfg_group[old_property_name]
+          new_cfg_group[new_property_name] = self._process_property_value_transformation(catalog,
map_property_item, old_cfg_group[old_property_name])
+          if new_property_name != old_property_name:
+            del old_cfg_group[old_property_name]
         elif old_property_name not in old_cfg_group and default_value is not None:
           new_cfg_group[new_property_name] = default_value
 
@@ -722,14 +773,27 @@ class ServerConfig(object):
       self._hash = self._calculate_hash()
     elif action == CatConst.ACTION_COMMIT:
       self._commit()
-    elif action == CatConst.ACTION_RENAME_PROPERTY and isinstance(arg, list) and len(arg)
== 2:
-      self._rename_property(arg[0], arg[1])
+    elif action == CatConst.ACTION_RENAME_PROPERTY and isinstance(arg, list) and len(arg)
== 5:
+      self._rename_property(*arg)
 
-  def _rename_property(self, old_name, new_name):
+  def _rename_property(self, old_name, new_name, transform_func, catalog, map_item):
+    """
+    :type old_name: str
+    :type new_name: str
+    :type transform_func: function
+    :type catalog: UpgradeCatalog
+    :type map_item: dict
+    :return:
+    """
     if old_name in self.properties:
       old_property_value = self.properties[old_name]
-      self.properties[new_name] = old_property_value
-      del self.properties[old_name]
+      if transform_func is not None:
+        self.properties[new_name] = transform_func(catalog, map_item, old_property_value)
+      else:
+        self.properties[new_name] = old_property_value
+
+      if old_name != new_name:
+        del self.properties[old_name]
 
   def is_attributes_exists(self):
     return CatConst.STACK_PROPERTIES_ATTRIBUTES in self._configs

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa206f75/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
index daa291c..c1c7970 100644
--- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.1_to_2.3.json
@@ -331,7 +331,10 @@
         }
       },
       "property-mapping": {
-        "nimbus.host": "nimbus.seeds",
+        "nimbus.host": {
+          "map-to": "nimbus.seeds",
+          "coerce-to": "yaml-array"
+        },
         "mapreduce.job.speculative.speculativecap": {
           "map-to": "mapreduce.job.speculative.speculative-cap-running-tasks",
           "from-catalog": "mapred-site",

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa206f75/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
index 5b10230..463b98e 100644
--- a/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
+++ b/ambari-server/src/main/resources/upgrade/catalog/UpgradeCatalog_2.2_to_2.3.json
@@ -1343,7 +1343,10 @@
                         "RANGER"
                     ]
                 },
-              "nimbus.host": "nimbus.seeds",
+              "nimbus.host": {
+                "map-to": "nimbus.seeds",
+                "coerce-to": "yaml-array"
+              },
               "mapreduce.job.speculative.speculativecap": {
                 "map-to": "mapreduce.job.speculative.speculative-cap-running-tasks",
                 "from-catalog": "mapred-site",

http://git-wip-us.apache.org/repos/asf/ambari/blob/fa206f75/ambari-server/src/test/python/TestUpgradeHelper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestUpgradeHelper.py b/ambari-server/src/test/python/TestUpgradeHelper.py
index 45b899e..835f39a 100644
--- a/ambari-server/src/test/python/TestUpgradeHelper.py
+++ b/ambari-server/src/test/python/TestUpgradeHelper.py
@@ -415,6 +415,127 @@ class TestUpgradeHelper(TestCase):
 
     self.assertEqual(expected_curl_args, curl_mock.call_args_list)
 
+  @patch.object(upgradeHelper, "get_config_resp_all")
+  def test_coerce_tag(self, get_config_resp_all_mock):
+    test_catalog = """
+        {
+      "version": "1.0",
+      "stacks": [
+        {
+          "name": "HDP",
+          "old-version": "1.0",
+          "target-version": "1.1",
+          "options": {
+            "config-types":{
+              "test": {
+                "merged-copy": "yes"
+              }
+            }
+          },
+          "properties": {
+             "test": {
+               "test": "host1.com"
+            }
+          },
+          "property-mapping": {
+            "test":{
+                "map-to": "test-arr",
+                "coerce-to": "yaml-array"
+           }
+          }
+        }
+      ]
+    }
+    """
+    old_opt = upgradeHelper.Options.OPTIONS
+    options = lambda: ""
+    options.from_stack = "1.0"
+    options.to_stack = "1.1"
+    options.upgrade_json = ""
+
+    upgradeHelper.Options.OPTIONS = options
+    upgradeHelper.Options.SERVICES = [self.required_service]
+    get_config_resp_all_mock.return_value = {
+      "test": {
+        "properties": {}
+      }
+    }
+
+    ucf = UpgradeCatalogFactoryMock(test_catalog)
+    scf = upgradeHelper.ServerConfigFactory()
+
+    cfg = scf.get_config("test")
+    ucfg = ucf.get_catalog("1.0", "1.1")
+
+    cfg.merge(ucfg)
+    scf.process_mapping_transformations(ucfg)
+
+    upgradeHelper.Options.OPTIONS = old_opt
+
+    self.assertEqual(True, "test-arr" in cfg.properties)
+    self.assertEqual("['host1.com']", cfg.properties["test-arr"])
+
+  @patch.object(upgradeHelper, "get_config_resp_all")
+  def test_replace_tag(self, get_config_resp_all_mock):
+    test_catalog = """
+        {
+      "version": "1.0",
+      "stacks": [
+        {
+          "name": "HDP",
+          "old-version": "1.0",
+          "target-version": "1.1",
+          "options": {
+            "config-types":{
+              "test": {
+                "merged-copy": "yes"
+              }
+            }
+          },
+          "properties": {
+             "test": {
+               "test": "host1.com"
+            }
+          },
+          "property-mapping": {
+            "test":{
+                "map-to": "test-arr",
+                "replace-from": "com",
+                "replace-to": "org"
+           }
+          }
+        }
+      ]
+    }
+    """
+    old_opt = upgradeHelper.Options.OPTIONS
+    options = lambda: ""
+    options.from_stack = "1.0"
+    options.to_stack = "1.1"
+    options.upgrade_json = ""
+
+    upgradeHelper.Options.OPTIONS = options
+    upgradeHelper.Options.SERVICES = [self.required_service]
+    get_config_resp_all_mock.return_value = {
+      "test": {
+        "properties": {}
+      }
+    }
+
+    ucf = UpgradeCatalogFactoryMock(test_catalog)
+    scf = upgradeHelper.ServerConfigFactory()
+
+    cfg = scf.get_config("test")
+    ucfg = ucf.get_catalog("1.0", "1.1")
+
+    cfg.merge(ucfg)
+    scf.process_mapping_transformations(ucfg)
+
+    upgradeHelper.Options.OPTIONS = old_opt
+
+    self.assertEqual(True, "test-arr" in cfg.properties)
+    self.assertEqual("host1.org", cfg.properties["test-arr"])
+
   @patch.object(upgradeHelper, "curl")
   @patch("time.time")
   def test_update_config(self, time_mock, curl_mock):
@@ -734,6 +855,7 @@ class TestUpgradeHelper(TestCase):
   @patch("__builtin__.open")
   def test_verify_configuration(self, open_mock, configuration_diff_analyze_mock, configuration_item_diff_mock,
                                 get_config_resp_all_mock, upgradecatalogfactory_mock):
+    old_opt = upgradeHelper.Options.OPTIONS
     options = lambda: ""
     options.from_stack = self.catalog_from
     options.to_stack = self.catalog_to
@@ -751,6 +873,8 @@ class TestUpgradeHelper(TestCase):
     # execute testing function
     upgradeHelper.verify_configuration()
 
+    upgradeHelper.Options.OPTIONS = old_opt
+
     self.assertEqual(1, get_config_resp_all_mock.call_count)
     self.assertEqual(1, configuration_item_diff_mock.call_count)
     self.assertEqual(1, configuration_diff_analyze_mock.call_count)


Mime
View raw message