ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [2/2] ambari git commit: AMBARI-8907 - Upgrade Pack for Falcon (jonathanhurley)
Date Mon, 29 Dec 2014 05:20:56 GMT
AMBARI-8907 - Upgrade Pack for Falcon (jonathanhurley)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4af27666
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4af27666
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4af27666

Branch: refs/heads/trunk
Commit: 4af27666afb0892a8403302ac95333393c026aeb
Parents: 9c73f2c
Author: Jonathan Hurley <jhurley@hortonworks.com>
Authored: Wed Dec 24 11:03:02 2014 -0500
Committer: Jonathan Hurley <jhurley@hortonworks.com>
Committed: Mon Dec 29 00:19:19 2014 -0500

----------------------------------------------------------------------
 .../0.5.0.2.1/package/scripts/falcon_client.py  |   16 +
 .../0.5.0.2.1/package/scripts/falcon_server.py  |   28 +-
 .../package/scripts/falcon_server_upgrade.py    |  102 ++
 .../FALCON/0.5.0.2.1/package/scripts/params.py  |   20 +-
 .../stacks/HDP/2.2/upgrades/upgrade-2.2.xml     |   22 +-
 .../stacks/2.1/FALCON/test_falcon_server.py     |   33 +-
 .../stacks/2.2/configs/falcon-upgrade.json      | 1093 ++++++++++++++++++
 7 files changed, 1307 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
index fd8c005..6f1cd56 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
@@ -25,14 +25,30 @@ class FalconClient(Script):
     self.install_packages(env)
     self.configure(env)
 
+
   def configure(self, env):
     import params
 
     env.set_params(params)
     falcon('client', action='config')
 
+
   def status(self, env):
     raise ClientComponentHasNoStatus()
 
+
+  def pre_rolling_restart(self, env):
+    import params
+    env.set_params(params)
+
+    # this function should not execute if the version can't be determined or
+    # is not at least HDP 2.2.0.0
+    if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
< 0:
+      return
+
+    Logger.info("Executing Falcon Client Rolling Upgrade pre-restart")
+    Execute(format("hdp-select set hadoop-client {version}"))
+
+
 if __name__ == "__main__":
   FalconClient().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
index 0460460..536bd49 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
@@ -17,7 +17,10 @@ limitations under the License.
 
 """
 
+import falcon_server_upgrade
+
 from resource_management import *
+from resource_management.libraries.functions.version import *
 from falcon import falcon
 
 class FalconServer(Script):
@@ -27,7 +30,8 @@ class FalconServer(Script):
     self.install_packages(env)
     env.set_params(params)
 
-  def start(self, env):
+
+  def start(self, env, rolling_restart=False):
     import params
 
     env.set_params(params)
@@ -35,13 +39,18 @@ class FalconServer(Script):
 
     falcon('server', action='start')
 
-  def stop(self, env):
+
+  def stop(self, env, rolling_restart=False):
     import params
 
     env.set_params(params)
 
     falcon('server', action='stop')
 
+    # if performing an upgrade, backup some directories after stopping falcon
+    if rolling_restart:
+      falcon_server_upgrade.post_stop_backup()
+
 
   def configure(self, env):
     import params
@@ -50,6 +59,7 @@ class FalconServer(Script):
 
     falcon('server', action='config')
 
+
   def status(self, env):
     import status_params
 
@@ -57,5 +67,19 @@ class FalconServer(Script):
     check_process_status(status_params.server_pid_file)
 
 
+  def pre_rolling_restart(self, env):
+    import params
+    env.set_params(params)
+
+    # this function should not execute if the version can't be determined or
+    # is not at least HDP 2.2.0.0
+    if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0')
< 0:
+      return
+
+    Logger.info("Executing Falcon Server Rolling Upgrade pre-restart")
+    Execute(format("hdp-select set falcon-server {version}"))
+    falcon_server_upgrade.pre_start_restore()
+
+
 if __name__ == "__main__":
   FalconServer().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
new file mode 100644
index 0000000..e6819d2
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
@@ -0,0 +1,102 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+import shutil
+import tarfile
+import tempfile
+
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+
+BACKUP_TEMP_DIR = "falcon-upgrade-backup"
+BACKUP_DATA_ARCHIVE = "falcon-local-backup.tar"
+BACKUP_CONF_ARCHIVE = "falcon-conf-backup.tar"
+
+def post_stop_backup():
+  """
+  Backs up the falcon configuration and data directories as part of the
+  upgrade process.
+  :return:
+  """
+  Logger.info('Backing up Falcon data and configuration directories before upgrade...')
+  directoryMappings = _get_directory_mappings()
+
+  absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
+  if not os.path.isdir(absolute_backup_dir):
+    os.makedirs(absolute_backup_dir)
+
+  for directory in directoryMappings:
+    if not os.path.isdir(directory):
+      raise Fail("Unable to backup missing directory {0}".format(directory))
+
+    archive = os.path.join(absolute_backup_dir, directoryMappings[directory])
+    Logger.info('Compressing {0} to {1}'.format(directory, archive))
+
+    if os.path.exists(archive):
+      os.remove(archive)
+
+    tarball = None
+    try:
+      tarball = tarfile.open(archive, "w")
+      tarball.add(directory, arcname=os.path.basename(directory))
+    finally:
+      if tarball:
+        tarball.close()
+
+
+def pre_start_restore():
+  """
+  Restores the data and configuration backups to their proper locations
+  after an upgrade has completed.
+  :return:
+  """
+  Logger.info('Restoring Falcon data and configuration directories after upgrade...')
+  directoryMappings = _get_directory_mappings()
+
+  for directory in directoryMappings:
+    archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
+      directoryMappings[directory])
+
+    if not os.path.isfile(archive):
+      raise Fail("Unable to restore missing backup archive {0}".format(archive))
+
+    Logger.info('Extracting {0} to {1}'.format(archive, directory))
+
+    tarball = None
+    try:
+      tarball = tarfile.open(archive, "r")
+      tarball.extractall(directory)
+    finally:
+      if tarball:
+        tarball.close()
+
+  # cleanup
+  shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
+
+
+def _get_directory_mappings():
+  """
+  Gets a dictionary of directory to archive name that represents the
+  directories that need to be backed up and their output tarball archive targets
+  :return:  the dictionary of directory to tarball mappings
+  """
+  import params
+
+  return { params.falcon_local_dir : BACKUP_DATA_ARCHIVE,
+    params.falcon_conf_dir : BACKUP_CONF_ARCHIVE }

http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
index 73258fe..82cd470 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
@@ -24,14 +24,28 @@ from status_params import *
 
 config = Script.get_config()
 
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
 stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
 hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
 
-#hadoop params
+# hadoop params
 if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
   hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
-  falcon_webapp_dir = "/usr/hdp/current/falcon-client/webapp"
-  falcon_home = "/usr/hdp/current/falcon-client"
+
+  # if this is a server action, then use the server binaries; smoke tests
+  # use the client binaries
+  server_role_dir_mapping = { 'FALCON_SERVER' : 'falcon-server',
+    'FALCON_SERVICE_CHECK' : 'falcon-client' }
+
+  command_role = default("/role", "")
+  if command_role not in server_role_dir_mapping:
+    command_role = 'FALCON_SERVICE_CHECK'
+
+  falcon_root = server_role_dir_mapping[command_role]
+  falcon_webapp_dir = format('/usr/hdp/current/{falcon_root}/webapp')
+  falcon_home = format('/usr/hdp/current/{falcon_root}')
 else:
   hadoop_bin_dir = "/usr/bin"
   falcon_webapp_dir = '/var/lib/falcon/webapp'

http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index 8d8fe00..dd80134 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -75,6 +75,13 @@
       </service>
     </group>
 
+    <group name="FALCON" title="Falcon">
+      <service name="Falcon">
+        <component>FALCON_SERVER</component>
+        <component>FALCON_CLIENT</component>
+      </service>
+    </group>
+
     <group name="CLIENTS" title="Client Components">
       <service name="HDFS">
         <component>HDFS_CLIENT</component>
@@ -125,7 +132,6 @@
         </task>
       </execute-stage>
     </group>
-
   </order>
   
   
@@ -365,5 +371,19 @@
         </upgrade>
       </component>
     </service>
+
+    <service name="FALCON">
+      <component name="FALCON_SERVER">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+      <component name="FALCON_CLIENT">
+        <upgrade>
+          <task xsi:type="restart" />
+        </upgrade>
+      </component>
+    </service>
+
   </processing>
 </upgrade>

http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
index 5451cf8..9b8579a 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
@@ -18,12 +18,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 
+from mock.mock import MagicMock, patch
 from stacks.utils.RMFTestCase import *
 
-
 class TestFalconServer(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "FALCON/0.5.0.2.1/package"
   STACK_VERSION = "2.1"
+  UPGRADE_STACK_VERSION = "2.2"
 
   def test_start_default(self):
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py",
@@ -142,3 +143,33 @@ class TestFalconServer(RMFTestCase):
                               )
 
 
+  @patch("shutil.rmtree", new = MagicMock())
+  @patch("tarfile.open")
+  @patch("os.path.isdir")
+  @patch("os.path.exists")
+  @patch("os.path.isfile")
+  def test_upgrade(self, isfile_mock, exists_mock, isdir_mock,
+      tarfile_open_mock):
+
+    isdir_mock.return_value = True
+    exists_mock.side_effect = [False,False,True]
+    isfile_mock.return_value = True
+
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py",
+     classname = "FalconServer", command = "restart", config_file = "falcon-upgrade.json",
+     hdp_stack_version = self.UPGRADE_STACK_VERSION,
+     target = RMFTestCase.TARGET_COMMON_SERVICES )
+
+    self.assertResourceCalled('Execute',
+      '/usr/hdp/current/falcon-server/bin/falcon-stop',
+      path = ['/usr/hdp/current/hadoop-client/bin'], user='falcon')
+
+    self.assertResourceCalled('File', '/var/run/falcon/falcon.pid',
+      action = ['delete'])
+
+    self.assertResourceCalled('Execute', 'hdp-select set falcon-server 2.2.1.0-2135')
+
+    # 4 calls to tarfile.open (2 directories * read + write)
+    self.assertTrue(tarfile_open_mock.called)
+    self.assertEqual(tarfile_open_mock.call_count,4)
+


Mime
View raw message