ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From alejan...@apache.org
Subject ambari git commit: AMBARI-13164. RU: Knox to use versioned data dir starting in HDP 2.3.2.0 (alejandro)
Date Mon, 21 Sep 2015 20:49:50 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1.2 8a59fdb96 -> e16b6bbd3


AMBARI-13164. RU: Knox to use versioned data dir starting in HDP 2.3.2.0 (alejandro)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e16b6bbd
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e16b6bbd
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e16b6bbd

Branch: refs/heads/branch-2.1.2
Commit: e16b6bbd385a61aa81e86e1a8a5b9d47e0c4e5f6
Parents: 8a59fdb
Author: Alejandro Fernandez <afernandez@hortonworks.com>
Authored: Mon Sep 21 13:48:07 2015 -0700
Committer: Alejandro Fernandez <afernandez@hortonworks.com>
Committed: Mon Sep 21 13:49:03 2015 -0700

----------------------------------------------------------------------
 .../libraries/script/script.py                  |  18 +-
 .../0.5.0.2.2/package/scripts/params_linux.py   |  35 +++-
 .../KNOX/0.5.0.2.2/package/scripts/upgrade.py   |  30 +++-
 .../package/scripts/nodemanager_upgrade.py      |   2 +-
 .../python/stacks/2.2/KNOX/test_knox_gateway.py | 164 ++++++++++++++++++-
 5 files changed, 232 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/e16b6bbd/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index b4731a0..f43c414 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -300,12 +300,26 @@ class Script(object):
     return format_hdp_stack_version(stack_version_unformatted)
 
   @staticmethod
+  def is_hdp_stack_greater(formatted_hdp_stack_version, compare_to_version):
+    """
+    Gets whether the provided formatted_hdp_stack_version (normalized)
+    is greater than the specified stack version
+    :param formatted_hdp_stack_version: the version of stack to compare
+    :param compare_to_version: the version of stack to compare to
+    :return: True if the command's stack is greater than the specified version
+    """
+    if formatted_hdp_stack_version is None or formatted_hdp_stack_version == "":
+      return False
+
+    return compare_versions(formatted_hdp_stack_version, compare_to_version) > 0
+
+  @staticmethod
   def is_hdp_stack_greater_or_equal(compare_to_version):
     """
     Gets whether the hostLevelParams/stack_version, after being normalized,
     is greater than or equal to the specified stack version
     :param compare_to_version: the version to compare to
-    :return: True if the command's stack is greater than the specified version
+    :return: True if the command's stack is greater than or equal the specified version
     """
     return Script.is_hdp_stack_greater_or_equal_to(Script.get_hdp_stack_version(), compare_to_version)
 
@@ -316,7 +330,7 @@ class Script(object):
     is greater than or equal to the specified stack version
     :param formatted_hdp_stack_version: the version of stack to compare
     :param compare_to_version: the version of stack to compare to
-    :return: True if the command's stack is greater than the specified version
+    :return: True if the command's stack is greater than or equal to the specified version
     """
     if formatted_hdp_stack_version is None or formatted_hdp_stack_version == "":
       return False

http://git-wip-us.apache.org/repos/asf/ambari/blob/e16b6bbd/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
index 5ff30c3..9c083f1 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params_linux.py
@@ -18,7 +18,8 @@ limitations under the License.
 Ambari Agent
 
 """
-from resource_management import *
+from resource_management.core.logger import Logger
+
 import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json
module and has the same functions set.
 from resource_management.libraries.functions import format
 from resource_management.libraries.functions.version import format_hdp_stack_version
@@ -35,9 +36,37 @@ tmp_dir = Script.get_tmp_dir()
 stack_name = default("/hostLevelParams/stack_name", None)
 upgrade_direction = default("/commandParams/upgrade_direction", None)
 version = default("/commandParams/version", None)
+# E.g., 2.3.2.0
+version_formatted = format_hdp_stack_version(version)
+
+# E.g., 2.3
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
+
+# This is the version whose state is CURRENT. During an RU, this is the source version.
+# DO NOT format it since we need the build number too.
+upgrade_from_version = default("/hostLevelParams/current_version", None)
+
+# server configurations
+# Default value used in HDP 2.3.0.0 and earlier.
+
+knox_data_dir = '/var/lib/knox/data'
+
+# Important, it has to be strictly greater than 2.3.0.0!!!
+if stack_name and stack_name.upper() == "HDP":
+  Logger.info(format("HDP version to use is {version_formatted}"))
+  if Script.is_hdp_stack_greater(version_formatted, "2.3.0.0"):
+    # This is the current version. In the case of a Rolling Upgrade, it will be the newer
version.
+    # In the case of a Downgrade, it will be the version downgrading to.
+    # This is always going to be a symlink to /var/lib/knox/data_${version}
+    knox_data_dir = format('/usr/hdp/{version}/knox/data')
+    Logger.info(format("Detected HDP with stack version {version}, will use knox_data_dir
= {knox_data_dir}"))
+
+
+knox_logs_dir = '/var/log/knox'
 
-knox_master_secret_path = '/var/lib/knox/data/security/master'
-knox_cert_store_path = '/var/lib/knox/data/security/keystores/gateway.jks'
+knox_master_secret_path = format('{knox_data_dir}/security/master')
+knox_cert_store_path = format('{knox_data_dir}/security/keystores/gateway.jks')
 knox_user = default("/configurations/knox-env/knox_user", "knox")
 
 # server configurations

http://git-wip-us.apache.org/repos/asf/ambari/blob/e16b6bbd/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
index 2c805fa..a9bb9e9 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/upgrade.py
@@ -25,6 +25,10 @@ import tempfile
 from resource_management.core.logger import Logger
 from resource_management.core.exceptions import Fail
 from resource_management.libraries.functions import tar_archive
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions import Direction
+from resource_management.libraries.functions.version import compare_versions,format_hdp_stack_version
+
 
 BACKUP_TEMP_DIR = "knox-upgrade-backup"
 BACKUP_DATA_ARCHIVE = "knox-data-backup.tar"
@@ -36,7 +40,7 @@ def backup_data():
   :return: Returns the path to the absolute backup directory.
   """
   Logger.info('Backing up Knox data directory before upgrade...')
-  directoryMappings = _get_directory_mappings()
+  directoryMappings = _get_directory_mappings_during_upgrade()
 
   absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
   if not os.path.isdir(absolute_backup_dir):
@@ -58,7 +62,7 @@ def backup_data():
   return absolute_backup_dir
 
 
-def _get_directory_mappings():
+def _get_directory_mappings_during_upgrade():
   """
   Gets a dictionary of directory to archive name that represents the
   directories that need to be backed up and their output tarball archive targets
@@ -66,6 +70,24 @@ def _get_directory_mappings():
   """
   import params
 
-  return { params.knox_data_dir : BACKUP_DATA_ARCHIVE,
-           params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE} # the trailing "/" is important
here so as to not include the "conf" folder itself
+  # Must be performing an Upgrade
+  if params.upgrade_direction is None or params.upgrade_direction != Direction.UPGRADE or
\
+          params.upgrade_from_version is None or params.upgrade_from_version == "":
+    Logger.error("Function _get_directory_mappings_during_upgrade() can only be called during
a Stack Upgrade in direction UPGRADE.")
+    return {}
+
+  # By default, use this for all stacks.
+  knox_data_dir = '/var/lib/knox/data'
+
+  if params.stack_name and params.stack_name.upper() == "HDP" and \
+          compare_versions(format_hdp_stack_version(params.upgrade_from_version), "2.3.0.0")
> 0:
+    # Use the version that is being upgraded from.
+    knox_data_dir = format('/usr/hdp/{upgrade_from_version}/knox/data')
+
+
+  directories = {knox_data_dir: BACKUP_DATA_ARCHIVE,
+                params.knox_conf_dir + "/": BACKUP_CONF_ARCHIVE} # the trailing "/" is important
here so as to not include the "conf" folder itself
+
+  Logger.info(format("Knox directories to backup:\n{directories}"))
+  return directories
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/e16b6bbd/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
index 9ac2d63..01f8349 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/nodemanager_upgrade.py
@@ -41,7 +41,7 @@ def post_upgrade_check():
   _check_nodemanager_startup()
 
 
-@retry(times=12, sleep_time=10, err_class=Fail)
+@retry(times=30, sleep_time=10, err_class=Fail)
 def _check_nodemanager_startup():
   '''
   Checks that a NodeManager is in a RUNNING state in the cluster via

http://git-wip-us.apache.org/repos/asf/ambari/blob/e16b6bbd/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
index 3ae42a0..83bba1f 100644
--- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
+++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
@@ -245,7 +245,7 @@ class TestKnoxGateway(RMFTestCase):
     self.assertResourceCalled('Execute', ('tar',
      '-zcvhf',
      '/tmp/knox-upgrade-backup/knox-data-backup.tar',
-     '/usr/hdp/current/knox-server/data/'),
+     '/var/lib/knox/data'),
         sudo = True,
     )
     self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', '2.2.1.0-3242'),
@@ -257,12 +257,15 @@ class TestKnoxGateway(RMFTestCase):
   @patch("os.path.exists")
   @patch("os.path.isdir")
   @patch("resource_management.core.shell.call")
-  def test_pre_rolling_restart_23(self, call_mock, isdir_mock, path_exists_mock, remove_mock):
+  def test_pre_rolling_restart_to_hdp_2300(self, call_mock, isdir_mock, path_exists_mock,
remove_mock):
+    """
+    In HDP 2.3.0.0, Knox was using a data dir of /var/lib/knox/data
+    """
     isdir_mock.return_value = True
     config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
     with open(config_file, "r") as f:
       json_content = json.load(f)
-    version = '2.3.0.0-1234'
+    version = "2.3.0.0-1234"
     json_content['commandParams']['version'] = version
 
     path_exists_mock.return_value = True
@@ -286,10 +289,10 @@ class TestKnoxGateway(RMFTestCase):
     self.assertResourceCalled('Execute', ('tar',
      '-zcvhf',
      '/tmp/knox-upgrade-backup/knox-data-backup.tar',
-     '/usr/hdp/current/knox-server/data/'),
+     '/var/lib/knox/data'),
         sudo = True,
     )
-    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', '2.3.0.0-1234'),
+    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', version),
         sudo = True,
     )
     self.assertResourceCalled('Execute', ('cp',
@@ -312,12 +315,159 @@ class TestKnoxGateway(RMFTestCase):
     self.assertEquals(1, mocks_dict['call'].call_count)
     self.assertEquals(1, mocks_dict['checked_call'].call_count)
     self.assertEquals(
-      ('conf-select', 'set-conf-dir', '--package', 'knox', '--stack-version', '2.3.0.0-1234',
'--conf-version', '0'),
+      ('conf-select', 'set-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version',
'0'),
        mocks_dict['checked_call'].call_args_list[0][0][0])
     self.assertEquals(
-      ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', '2.3.0.0-1234',
'--conf-version', '0'),
+      ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', version,
'--conf-version', '0'),
        mocks_dict['call'].call_args_list[0][0][0])
 
+  @patch("os.remove")
+  @patch("os.path.exists")
+  @patch("os.path.isdir")
+  @patch("resource_management.core.shell.call")
+  def test_pre_rolling_restart_from_hdp_2300_to_2320(self, call_mock, isdir_mock, path_exists_mock,
remove_mock):
+    """
+    In RU from HDP 2.3.0.0 to 2.3.2.0, should backup the data dir used by the source version,
which
+    is /var/lib/knox/data
+    """
+    isdir_mock.return_value = True
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    source_version = "2.3.0.0-1234"
+    version = "2.3.2.0-5678"
+    # This is an RU from 2.3.0.0 to 2.3.2.0
+    json_content['commandParams']['version'] = version
+    json_content['hostLevelParams']['current_version'] = source_version
+
+    path_exists_mock.return_value = True
+    mocks_dict = {}
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
+                       classname = "KnoxGateway",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES,
+                       call_mocks = [(0, None), (0, None)],
+                       mocks_dict = mocks_dict)
+
+    self.assertResourceCalled('Execute', ('tar',
+     '-zcvhf',
+     '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+     '/usr/hdp/current/knox-server/conf/'),
+        sudo = True,
+    )
+    self.assertResourceCalled('Execute', ('tar',
+     '-zcvhf',
+     '/tmp/knox-upgrade-backup/knox-data-backup.tar',
+     '/var/lib/knox/data'),
+        sudo = True,
+    )
+    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', version),
+        sudo = True,
+    )
+    self.assertResourceCalled('Execute', ('cp',
+     '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+     '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar'),
+        sudo = True,
+    )
+    self.assertResourceCalled('Execute', ('tar',
+     '-xvf',
+     '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+     '-C',
+     '/usr/hdp/current/knox-server/conf/'),
+        sudo = True,
+    )
+    self.assertResourceCalled('File', '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar',
+        action = ['delete'],
+    )
+    self.assertNoMoreResources()
+
+    self.assertEquals(1, mocks_dict['call'].call_count)
+    self.assertEquals(1, mocks_dict['checked_call'].call_count)
+    self.assertEquals(
+      ('conf-select', 'set-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version',
'0'),
+       mocks_dict['checked_call'].call_args_list[0][0][0])
+    self.assertEquals(
+      ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', version,
'--conf-version', '0'),
+       mocks_dict['call'].call_args_list[0][0][0])
+
+  @patch("os.remove")
+  @patch("os.path.exists")
+  @patch("os.path.isdir")
+  @patch("resource_management.core.shell.call")
+  def test_pre_rolling_restart_from_hdp_2320(self, call_mock, isdir_mock, path_exists_mock,
remove_mock):
+    """
+    In RU from HDP 2.3.2 to anything higher, should backup the data dir used by the source
version, which
+    is /var/lib/knox/data_${source_version}
+    """
+    isdir_mock.return_value = True
+    config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/knox_upgrade.json"
+    with open(config_file, "r") as f:
+      json_content = json.load(f)
+    source_version = "2.3.2.0-1000"
+    version = "2.3.2.0-1001"
+    # This is an RU from 2.3.2.0 to 2.3.2.1
+    json_content['commandParams']['version'] = version
+    json_content['hostLevelParams']['current_version'] = source_version
+
+    path_exists_mock.return_value = True
+    mocks_dict = {}
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
+                       classname = "KnoxGateway",
+                       command = "pre_rolling_restart",
+                       config_dict = json_content,
+                       hdp_stack_version = self.STACK_VERSION,
+                       target = RMFTestCase.TARGET_COMMON_SERVICES,
+                       call_mocks = [(0, None), (0, None)],
+                       mocks_dict = mocks_dict)
+
+    self.assertResourceCalled('Execute', ('tar',
+     '-zcvhf',
+     '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+     '/usr/hdp/current/knox-server/conf/'),
+        sudo = True,
+    )
+    self.assertResourceCalled('Execute', ('tar',
+     '-zcvhf',
+     '/tmp/knox-upgrade-backup/knox-data-backup.tar',
+     "/usr/hdp/%s/knox/data" % source_version),
+        sudo = True,
+    )
+
+    '''
+    self.assertResourceCalled('Execute', ('hdp-select', 'set', 'knox-server', version),
+        sudo = True,
+    )
+    self.assertResourceCalled('Execute', ('cp',
+     '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+     '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar'),
+        sudo = True,
+    )
+    self.assertResourceCalled('Execute', ('tar',
+     '-xvf',
+     '/tmp/knox-upgrade-backup/knox-conf-backup.tar',
+     '-C',
+     '/usr/hdp/current/knox-server/conf/'),
+        sudo = True,
+    )
+    self.assertResourceCalled('File', '/usr/hdp/current/knox-server/conf/knox-conf-backup.tar',
+        action = ['delete'],
+    )
+    self.assertNoMoreResources()
+
+    self.assertEquals(1, mocks_dict['call'].call_count)
+    self.assertEquals(1, mocks_dict['checked_call'].call_count)
+    self.assertEquals(
+      ('conf-select', 'set-conf-dir', '--package', 'knox', '--stack-version', version, '--conf-version',
'0'),
+       mocks_dict['checked_call'].call_args_list[0][0][0])
+    self.assertEquals(
+      ('conf-select', 'create-conf-dir', '--package', 'knox', '--stack-version', version,
'--conf-version', '0'),
+       mocks_dict['call'].call_args_list[0][0][0])
+    '''
+
   @patch("os.path.islink")
   def test_start_default(self, islink_mock):
 


Mime
View raw message