ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject [ambari] branch branch-feature-AMBARI-14714 updated: [AMBARI-23121] Fix few instance manager issues found during ZK deploment (dsen) (#518)
Date Fri, 02 Mar 2018 13:30:17 GMT
This is an automated email from the ASF dual-hosted git repository.

dsen pushed a commit to branch branch-feature-AMBARI-14714
in repository https://gitbox.apache.org/repos/asf/ambari.git


The following commit(s) were added to refs/heads/branch-feature-AMBARI-14714 by this push:
     new d7bcc61  [AMBARI-23121] Fix few instance manager issues found during ZK deploment
(dsen) (#518)
d7bcc61 is described below

commit d7bcc61b68539612c4936669b305223b91623ad4
Author: Dmitry Sen <dsen@apache.org>
AuthorDate: Fri Mar 2 15:30:14 2018 +0200

    [AMBARI-23121] Fix few instance manager issues found during ZK deploment (dsen) (#518)
    
    * [AMBARI-23121] Fix few instance manager issues found during ZK deployment (dsen)
    
    * [AMBARI-23121] Fix few instance manager issues found during ZK deployment (dsen) - changes
according to review
---
 .../python/instance_manager/instance_manager.py    | 45 +++++++++++++++-------
 .../instance_manager/mpack-instance-manager.py     | 17 ++++++--
 .../instance_manager/test_instance_manager.py      | 43 ++++++++++++++-------
 3 files changed, 74 insertions(+), 31 deletions(-)

diff --git a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
index 6682234..c9f610b 100644
--- a/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
+++ b/mpack-instance-manager/src/main/python/instance_manager/instance_manager.py
@@ -51,8 +51,8 @@ def create_mpack(mpack_name, mpack_version, mpack_instance, subgroup_name=DEFAUL
   OR
   list of 'components_instances_name' to be created. (default) OR '*' for all components
   """
-  mpack_name = mpack_name.lower()
-  module_name = module_name.lower()
+  mpack_name, module_name, components, components_map = normalize_parameters(
+    mpack_name, module_name, components, components_map)
 
   validate_mpack_for_creation_or_changing(mpack_name, mpack_version, module_name, components,
components_map)
 
@@ -71,14 +71,17 @@ def set_mpack_instance(mpack, mpack_version, mpack_instance, subgroup_name=DEFAU
   OR
   list of 'components_instances_name' to be created. (default) OR '*' for all components
   """
-  mpack = mpack.lower()
-  module_name = module_name.lower()
+  mpack, module_name, components, components_map = normalize_parameters(
+    mpack, module_name, components, components_map)
 
   instances = MpackInstance.parse_instances_with_filtering(os.path.join(ROOT_FOLDER_PATH,
INSTANCES_FOLDER_NAME), mpack,
                                                            mpack_instance, subgroup_name,
module_name,
                                                            components, components_map)
   if not instances:
-    raise ValueError("Found no instances for the given filters.")
+    raise ValueError("Found no instances for the given filters: mpack_name:{0}, instance_name:{1},"
+                     " subgroup_name:{2}, module_name:{3}, components:{4}, components_map:{5}".
+                     format(mpack, mpack_instance, subgroup_name, module_name,
+                            components, components_map))
 
   validate_mpack_for_creation_or_changing(mpack, mpack_version, module_name, components,
components_map)
 
@@ -142,11 +145,8 @@ def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter,
 
   The output_conf_dir or output_path for each component instance will be included in json
depending on given parameters.
   """
-
-  if mpack_name_filter:
-    mpack_name_filter = mpack_name_filter.lower()
-  if module_name_filter:
-    module_name_filter = module_name_filter.lower()
+  mpack_name_filter, module_name_filter, skipped, components_name_filter_map = normalize_parameters(
+    mpack_name_filter, module_name_filter, [], components_name_filter_map)
 
   instances = MpackInstance.parse_instances_with_filtering(os.path.join(ROOT_FOLDER_PATH,
INSTANCES_FOLDER_NAME),
                                                            mpack_name_filter,
@@ -155,7 +155,10 @@ def build_granular_json_with_filtering(mpack_name_filter, instance_name_filter,
                                                            None,
                                                            components_name_filter_map)
   if not instances:
-    raise ValueError("Found no instances for the given filters.")
+    raise ValueError("Found no instances for the given filters: mpack_name:{0}, instance_name:{1},"
+                     " subgroup_name:{2}, module_name:{3}, components_name_map:{4}".
+                     format(mpack_name_filter, instance_name_filter, subgroup_name_filter,
+                            module_name_filter, components_name_filter_map))
 
   full_json_output = build_json_output(instances, output_conf_dir=output_conf_dir, output_path=output_path)
 
@@ -229,6 +232,22 @@ def build_json_output(instances, output_conf_dir=False, output_path=False):
   return {'mpacks': result}
 
 
+def normalize_parameters(mpack_name, module_name, components, components_map):
+  """
+  normalize parameters to be lowercase, for components_map dictionary only keys are normalized
+  """
+  if mpack_name:
+    mpack_name = mpack_name.lower()
+  if module_name:
+    module_name = module_name.lower()
+  if components and components != "*":
+    components = [component.lower() for component in components]
+  if components_map:
+    components_map = dict((k.lower(), v) for k, v in components_map.iteritems())
+
+  return mpack_name, module_name, components, components_map
+
+
 def validate_mpack_for_creation_or_changing(mpack_name, mpack_version, module_name, components,
components_map):
   mpack_root_path = os.path.join(ROOT_FOLDER_PATH, MPACKS_FOLDER_NAME, mpack_name)
   if not os.path.exists(mpack_root_path):
@@ -478,7 +497,7 @@ class ModuleInstance(Instance):
                 (components_name_filter_map and folder_name in components_name_filter_map)):
             components_map = ComponentInstance(name=DEFAULT_COMPONENT_INSTANCE_NAME,
                                                component_path=os.path.join(path, folder_name),
-                                               path_exec=os.path.realpath(
+                                               path_exec=os.readlink(
                                                  os.path.join(path, folder_name, CURRENT_SOFTLINK_NAME)))
         else:
           components_map = ComponentInstance.parse_into_components_dict(os.path.join(path,
folder_name),
@@ -566,7 +585,7 @@ class ComponentInstance(Instance):
       if not component_names_filter or component_instance_name in component_names_filter:
         result[component_instance_name] = ComponentInstance(name=component_instance_name,
                                                             component_path=os.path.join(path,
component_instance_name),
-                                                            path_exec=os.path.realpath(
+                                                            path_exec=os.readlink(
                                                               os.path.join(path, component_instance_name,
                                                                            CURRENT_SOFTLINK_NAME)))
     return result
diff --git a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
index 38c29f8..f9458b1 100644
--- a/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
+++ b/mpack-instance-manager/src/main/python/instance_manager/mpack-instance-manager.py
@@ -56,8 +56,14 @@ def check_required_options(options, action, parser):
     if not options.module_name:
       missing_options.append("module-name")
 
-  if missing_options:
-    parser.error("Missing following required command options: {0}".format(missing_options))
+    if missing_options:
+      parser.error("Missing following required command options: {0}".format(missing_options))
+
+    if not options.components_map and not options.components:
+      parser.error("Either components or components-map option must be specified.")
+
+    if options.components_map and options.components:
+      parser.error("Only components or components-map option could be specified. Can't use
both.")
 
 
 def init_create_parser_options(parser):
@@ -117,8 +123,11 @@ def main(options, args):
   parsed_components = None
   parsed_components_map = None
   try:
-    if hasattr(options, 'components') and options.components and options.components != '*':
-      parsed_components = ast.literal_eval(options.components)
+    if hasattr(options, 'components') and options.components:
+      if options.components == '*':
+        parsed_components = '*'
+      else:
+        parsed_components = ast.literal_eval(options.components)
     if options.components_map:
       parsed_components_map = ast.literal_eval(options.components_map)
   except ValueError:
diff --git a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
index 5315094..0d834cc 100644
--- a/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
+++ b/mpack-instance-manager/src/test/python/instance_manager/test_instance_manager.py
@@ -84,7 +84,7 @@ class TestInstanceManager(TestCase):
                                                 instance_manager.DEFAULT_MPACK_INSTANCE_NAME)))
 
   def test_create_mpack_server_module_with_multiple_component_instances(self):
-    create_mpack_with_defaults(components=None, components_map={SERVER_COMPONENT_NAME: ['server1',
'server2']})
+    create_mpack_with_defaults(components=None, components_map={SERVER_COMPONENT_NAME.upper():
['server1', 'server2']})
 
     current_link_1 = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME,
MPACK_NAME,
                                   INSTANCE_NAME_1, SUBGROUP_NAME, SERVER_MODULE_NAME, SERVER_COMPONENT_NAME,
@@ -121,12 +121,12 @@ class TestInstanceManager(TestCase):
                                   MPACK_VERSION_2, SERVER_COMPONENT_NAME))
 
   def test_set_version_client_module_asterisk(self):
-    create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
+    create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
 
     build_rpm_structure(mpack_version=MPACK_VERSION_2, remove_old_content=False, create_modules=False)
 
-    instance_manager.set_mpack_instance(MPACK_NAME, MPACK_VERSION_2, INSTANCE_NAME_1, SUBGROUP_NAME,
-                                        CLIENT_MODULE_NAME, '*')
+    instance_manager.set_mpack_instance(MPACK_NAME.upper(), MPACK_VERSION_2, INSTANCE_NAME_1,
SUBGROUP_NAME,
+                                        CLIENT_MODULE_NAME.upper(), '*')
 
     current_link = os.path.join(TMP_ROOT_FOLDER, instance_manager.INSTANCES_FOLDER_NAME,
MPACK_NAME,
                                 INSTANCE_NAME_1, SUBGROUP_NAME, CLIENT_COMPONENT_NAME,
@@ -160,9 +160,9 @@ class TestInstanceManager(TestCase):
                                   MPACK_VERSION_2, SERVER_COMPONENT_NAME))
 
   def test_get_conf_dir_all(self):
-    create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
-    create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None,
-                               components_map={SERVER_COMPONENT_NAME: ['server1']})
+    create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
+    create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
+                               components_map={SERVER_COMPONENT_NAME.upper(): ['server1']})
 
     conf_dir_json = instance_manager.get_conf_dir()
 
@@ -210,9 +210,9 @@ class TestInstanceManager(TestCase):
     self.assertEqual(conf_dir_json, expected_json)
 
   def test_list_instances_all(self):
-    create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME)
-    create_mpack_with_defaults(module_name=SERVER_MODULE_NAME, components=None,
-                               components_map={SERVER_COMPONENT_NAME: ['server1']})
+    create_mpack_with_defaults(module_name=CLIENT_MODULE_NAME.upper())
+    create_mpack_with_defaults(module_name=SERVER_MODULE_NAME.upper(), components=None,
+                               components_map={SERVER_COMPONENT_NAME.upper(): ['server1']})
 
     conf_dir_json = instance_manager.list_instances()
 
@@ -232,7 +232,7 @@ class TestInstanceManager(TestCase):
                         "hdfs_server": {
                           "component-instances": {
                             "server1": {
-                              "path": "/tmp/instance_manager_test/modules/hdfs/3.1.0.0-b1",
+                              "path": "/tmp/instance_manager_test/mpacks/hdpcore/1.0.0-b1/hdfs_server",
                               "name": "server1"
                             }
                           }
@@ -243,7 +243,7 @@ class TestInstanceManager(TestCase):
                       "category": "CLIENT",
                       "component_instances": {
                         "default": {
-                          "path": "/tmp/instance_manager_test/modules/hdfs-clients/3.1.0.0-b1",
+                          "path": "/tmp/instance_manager_test/mpacks/hdpcore/1.0.0-b1/hdfs_client",
                           "name": "default"
                         }
                       },
@@ -317,7 +317,7 @@ class TestInstanceManager(TestCase):
     expected_filter_result = {'mpacks': {'edw': {'mpack-instances': {'eCommerce': {'name':
'eCommerce', 'subgroups': {
       'default': {'modules': {'hdfs': {'category': 'SERVER', 'name': 'hdfs', 'components':
{'hdfs_server': {
         'component-instances': {
-          'server2': {'path': '/tmp/instance_manager_test/modules/hdfs/3.1.0.0-b1', 'name':
'server2'}}}}}}}}}}}}}
+          'server2': {'path': '/tmp/instance_manager_test/mpacks/edw/1.0.0-b1/hdfs_server',
'name': 'server2'}}}}}}}}}}}}}
     self.assertEquals(expected_filter_result, filter_by_component_instance_name_json)
 
   def test_validation(self):
@@ -363,6 +363,19 @@ class TestInstanceManager(TestCase):
                         "The instance /tmp/instance_manager_test/instances/hdpcore/Production/default/hdfs/"
                         "hdfs_server/default already exist. To change the version use set-mpack-instance
command")
 
+  def test_normalize_parameters(self):
+    mpack_name = MPACK_NAME.upper()
+    module_name = SERVER_MODULE_NAME.upper()
+    components = [SERVER_COMPONENT_NAME.upper()]
+    components_map = {SERVER_COMPONENT_NAME.upper(): ["DEFAULT"]}
+    mpack_name, module_name, components, components_map = instance_manager.normalize_parameters(
+      mpack_name, module_name, components, components_map)
+
+    self.assertEquals(mpack_name, MPACK_NAME.lower())
+    self.assertEquals(module_name, SERVER_MODULE_NAME.lower())
+    self.assertEquals(components, [SERVER_COMPONENT_NAME.lower()])
+    self.assertEquals(components_map, {SERVER_COMPONENT_NAME.lower(): ["DEFAULT"]})
+
   def test_set_non_existing_instance(self):
     try:
       instance_manager.set_mpack_instance(mpack=MPACK_NAME, mpack_version=MPACK_VERSION_1,
@@ -383,7 +396,9 @@ class TestInstanceManager(TestCase):
       raise AssertionError("The previous call should have thrown exception")
     except ValueError as e:
       self.assertEquals(e.message,
-                        "Found no instances for the given filters.")
+                        "Found no instances for the given filters: mpack_name:hdpcore, instance_name:Production,"
+                        " subgroup_name:default, module_name:hdfs, components:None,"
+                        " components_map:{'hdfs_server': ['non-existing-instance']}")
 
 
 def create_mpack_with_defaults(mpack_name=MPACK_NAME, mpack_version=MPACK_VERSION_1, mpack_instance=INSTANCE_NAME_1,

-- 
To stop receiving notification emails like this one, please contact
dsen@apache.org.

Mime
View raw message