ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hapyles...@apache.org
Subject [1/2] ambari git commit: AMBARI-21928 In case if component package doesn't advertise his version, Ambari silently failing on deploy stack steps (dgrinenko)
Date Tue, 12 Sep 2017 12:15:15 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.6 8c5007469 -> e7e58bbf4
  refs/heads/trunk 252e9898c -> 001f290be


AMBARI-21928 In case if component package doesn't advertise his version, Ambari silently failing
on deploy stack steps (dgrinenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/001f290b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/001f290b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/001f290b

Branch: refs/heads/trunk
Commit: 001f290be2622375786f27cc054727adb7be8905
Parents: 252e989
Author: Dmytro Grinenko <hapylestat@apache.org>
Authored: Tue Sep 12 15:12:44 2017 +0300
Committer: Dmytro Grinenko <hapylestat@apache.org>
Committed: Tue Sep 12 15:12:44 2017 +0300

----------------------------------------------------------------------
 .../libraries/functions/stack_tools.py          | 12 ++---
 .../libraries/functions/version_select_util.py  | 57 +++++++++-----------
 .../libraries/script/script.py                  |  2 +
 .../scripts/shared_initialization.py            | 19 +++----
 .../hooks/after-INSTALL/test_after_install.py   |  2 +-
 5 files changed, 42 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
index fa97fd1..d9233a3 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
@@ -32,6 +32,7 @@ from resource_management.core.utils import pad
 STACK_SELECTOR_NAME = "stack_selector"
 CONF_SELECTOR_NAME = "conf_selector"
 
+
 def get_stack_tool(name):
   """
   Give a tool selector name get the stack-specific tool name, tool path, tool package
@@ -43,7 +44,7 @@ def get_stack_tool(name):
   stack_name = default("/hostLevelParams/stack_name", None)
   if stack_name is None:
     Logger.warning("Cannot find the stack name in the command. Stack tools cannot be loaded")
-    return (None, None, None)
+    return None, None, None
 
   stack_tools = None
   stack_tools_config = default("/configurations/cluster-env/stack_tools", None)
@@ -52,23 +53,22 @@ def get_stack_tool(name):
 
   if stack_tools is None:
     Logger.warning("The stack tools could not be found in cluster-env")
-    return (None, None, None)
+    return None, None, None
 
   if stack_name not in stack_tools:
     Logger.warning("Cannot find stack tools for the stack named {0}".format(stack_name))
-    return (None, None, None)
+    return None, None, None
 
   # load the stack tooks keyed by the stack name
   stack_tools = stack_tools[stack_name]
 
   if not stack_tools or not name or name.lower() not in stack_tools:
     Logger.warning("Cannot find config for {0} stack tool in {1}".format(str(name), str(stack_tools)))
-    return (None, None, None)
-
+    return None, None, None
 
   tool_config = stack_tools[name.lower()]
 
-  # Return fixed length (tool_name, tool_path tool_package) tuple
+  # Return fixed length (tool_name, tool_path, tool_package) tuple
   return tuple(pad(tool_config[:3], 3))
 
 def get_stack_tool_name(name):

http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
index 79dc874..9fbb42b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
@@ -26,7 +26,6 @@ import tempfile
 from resource_management.core.logger import Logger
 from resource_management.core import shell
 from resource_management.libraries.functions import stack_tools
-import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json
module and has the same functions set.
 
 
 def get_component_version(stack_name, component_name):
@@ -39,42 +38,36 @@ def get_component_version(stack_name, component_name):
   :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
   """
   version = None
-  if stack_name is None or component_name is None:
+  if not stack_name or not component_name:
     Logger.error("Could not determine component version because of the parameters is empty.
" \
                  "stack_name: %s, component_name: %s" % (str(stack_name), str(component_name)))
     return version
 
-  out = None
-  code = -1
-  if not stack_name:
-    Logger.error("Stack name not provided")
-  elif not component_name:
-    Logger.error("Component name not provided")
+  stack_selector_name, stack_selector_path, stack_selector_package = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
+  if stack_selector_name and stack_selector_path and os.path.exists(stack_selector_path):
+    tmpfile = tempfile.NamedTemporaryFile(delete=True)
+    out, code = None, -1
+    get_stack_comp_version_cmd = '%s status %s > %s' % (stack_selector_path, component_name,
tmpfile.name)
+
+    try:
+      # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
+      # ToDo: original problem looks strange
+      with open(tmpfile.name, 'r') as f:
+        code = shell.call(get_stack_comp_version_cmd, quiet=True)[0]
+        out = f.read()
+
+      if code != 0 or out is None:
+        raise ValueError("Code is nonzero or output is empty")
+
+      Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
+      matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
+      version = matches[0][0].strip() if matches and len(matches) > 0 and len(matches[0])
> 0 else None
+      Logger.debug("Version for component %s: %s" % (component_name, str(version)))
+    except Exception, e:
+      Logger.error("Could not determine stack version for component %s by calling '%s'. Return
Code: %s, Output: %s." %
+                   (component_name, get_stack_comp_version_cmd, str(code), str(out)))
   else:
-    (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
-    if stack_selector_name and stack_selector_path and os.path.exists(stack_selector_path):
-      tmpfile = tempfile.NamedTemporaryFile()
-
-      get_stack_comp_version_cmd = ""
-      try:
-        # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
-        with open(tmpfile.name, 'r') as file:
-          get_stack_comp_version_cmd = '%s status %s > %s' % (stack_selector_path, component_name,
tmpfile.name)
-          code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True)
-          out = file.read()
-
-        if code != 0 or out is None:
-          raise Exception("Code is nonzero or output is empty")
-
-        Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
-        matches = re.findall(r"( [\d\.]+(\-\d+)?)", out)
-        version = matches[0][0].strip() if matches and len(matches) > 0 and len(matches[0])
> 0 else None
-        Logger.debug("Version for component %s: %s" % (component_name, str(version)))
-      except Exception, e:
-        Logger.error("Could not determine stack version for component %s by calling '%s'.
Return Code: %s, Output: %s." %
-                     (component_name, get_stack_comp_version_cmd, str(code), str(out)))
-    else:
-      Logger.error("Could not find stack selector for stack: %s" % str(stack_name))
+    Logger.error("Could not find stack selector for stack: %s" % str(stack_name))
 
   return version
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index fcfac2b..0e352d1 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -228,6 +228,8 @@ class Script(object):
         repo_version_id = default("/hostLevelParams/repository_version_id", None)
         if repo_version_id:
           self.put_structured_out({"repository_version_id": repo_version_id})
+      else:
+        Logger.error("Component '{0}' did not advertise a version. This may indicate a problem
with the component packaging.".format(stack_select_package_name))
 
 
   def should_expose_component_version(self, command_name):

http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
index daaffd8..67c3ba8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -46,7 +46,7 @@ def setup_stack_symlinks(struct_out_file):
     return
 
   if params.host_sys_prepped:
-    Logger.warning("Skipping running stack-selector-tool becase this is a sys_prepped host.
This may cause symlink pointers not to be created for HDP componets installed later on top
of an already sys_prepped host.")
+    Logger.warning("Skipping running stack-selector-tool because this is a sys_prepped host.
This may cause symlink pointers not to be created for HDP components installed later on top
of an already sys_prepped host")
     return
 
   # get the packages which the stack-select tool should be used on
@@ -105,17 +105,14 @@ def load_version(struct_out_file):
   """
   Load version from file.  Made a separate method for testing
   """
-  json_version = None
   try:
-    if os.path.exists(struct_out_file):
-      with open(struct_out_file, 'r') as fp:
-        json_info = json.load(fp)
-        json_version = json_info['version']
-  except:
-    pass
-
-  return json_version
-  
+    with open(struct_out_file, 'r') as fp:
+      json_info = json.load(fp)
+
+    return json_info['version']
+  except (IOError, KeyError, TypeError):
+    return None
+
 
 def link_configs(struct_out_file):
   """

http://git-wip-us.apache.org/repos/asf/ambari/blob/001f290b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
index d6740cb..f905cf9 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
@@ -344,4 +344,4 @@ class TestHookAfterInstall(RMFTestCase):
                        config_dict = json_content,
                        config_overrides = self.CONFIG_OVERRIDES)
 
-    logger_warning_mock.assert_any_call('Skipping running stack-selector-tool becase this
is a sys_prepped host. This may cause symlink pointers not to be created for HDP componets
installed later on top of an already sys_prepped host.')
+    logger_warning_mock.assert_any_call('Skipping running stack-selector-tool because this
is a sys_prepped host. This may cause symlink pointers not to be created for HDP components
installed later on top of an already sys_prepped host')


Mime
View raw message