ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From smoha...@apache.org
Subject git commit: AMBARI-3180. Helper script for HDP upgrade (update to script and unit test).
Date Wed, 11 Sep 2013 18:32:42 GMT
Updated Branches:
  refs/heads/trunk 66d717a03 -> 59076a55f


AMBARI-3180. Helper script for HDP upgrade (update to script and unit test).


Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/59076a55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/59076a55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/59076a55

Branch: refs/heads/trunk
Commit: 59076a55fe2e149a4a7018e42d400b56265ae98f
Parents: 66d717a
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Wed Sep 11 11:32:29 2013 -0700
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Wed Sep 11 11:32:29 2013 -0700

----------------------------------------------------------------------
 .../src/main/python/UpgradeHelper_HDP2.py       |  43 +++---
 .../src/test/python/TestUpgradeScript_HDP2.py   | 141 +++++++++----------
 2 files changed, 94 insertions(+), 90 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/59076a55/ambari-server/src/main/python/UpgradeHelper_HDP2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/python/UpgradeHelper_HDP2.py b/ambari-server/src/main/python/UpgradeHelper_HDP2.py
index 82974cc..20db21f 100644
--- a/ambari-server/src/main/python/UpgradeHelper_HDP2.py
+++ b/ambari-server/src/main/python/UpgradeHelper_HDP2.py
@@ -349,7 +349,7 @@ def get_mr1_mapping(options):
   hostmapping = {}
   for component in components:
     hostlist = []
-    response = curl('-u',
+    response = curl(False, '-u',
                     AUTH_FORMAT.format(options.user, options.password),
                     GET_URL_FORMAT.format(options.hostname, options.clustername, component))
     retcode, errdata = validate_response(response, True)
@@ -405,7 +405,7 @@ def delete_mr(options):
   for key, value in hostmapping.items():
     if (key in NON_CLIENTS) and (len(value) > 0):
       for host in value:
-        response = curl('-u',
+        response = curl(options.printonly, '-u',
                         AUTH_FORMAT.format(options.user, options.password),
                         '-X', 'PUT', '-d',
                         PUT_IN_MAINTENANCE,
@@ -418,7 +418,7 @@ def delete_mr(options):
     pass
   pass
 
-  response = curl('-u',
+  response = curl(options.printonly, '-u',
                   AUTH_FORMAT.format(options.user, options.password),
                   '-X', 'DELETE',
                   SERVICE_URL_FORMAT.format(options.hostname, options.clustername))
@@ -444,7 +444,7 @@ def add_services(options):
   hostmapping = read_mapping()
 
   for service in service_comp.keys():
-    response = curl('-u',
+    response = curl(options.printonly, '-u',
                     AUTH_FORMAT.format(options.user, options.password),
                     '-X', 'POST',
                     SERVICE_URL_FORMAT.format(options.hostname, options.clustername, service))
@@ -452,7 +452,7 @@ def add_services(options):
     if not retcode == 0:
       raise FatalException(retcode, errdata)
     for component in service_comp[service]:
-      response = curl('-u',
+      response = curl(options.printonly, '-u',
                       AUTH_FORMAT.format(options.user, options.password),
                       '-X', 'POST',
                       COMPONENT_URL_FORMAT.format(options.hostname, options.clustername,
service, component))
@@ -460,7 +460,7 @@ def add_services(options):
       if not retcode == 0:
         raise FatalException(retcode, errdata)
       for host in hostmapping[new_old_host_map[component]]:
-        response = curl('-u',
+        response = curl(options.printonly, '-u',
                         AUTH_FORMAT.format(options.user, options.password),
                         '-X', 'POST',
                         HOST_COMPONENT_URL_FORMAT.format(options.hostname, options.clustername,
host, component))
@@ -476,7 +476,7 @@ def add_services(options):
 def update_config(options, properties, type):
   tag = "version" + str(int(time.time() * 1000))
   properties_payload = {"Clusters": {"desired_config": {"type": type, "tag": tag, "properties":
properties}}}
-  response = curl('-u',
+  response = curl(options.printonly, '-u',
                   AUTH_FORMAT.format(options.user, options.password),
                   '-X', 'PUT', '-d',
                   json.dumps(properties_payload),
@@ -503,7 +503,7 @@ def get_config(options, type):
 
 def get_config_resp(options, type, error_if_na=True):
   CONFIG_URL_FORMAT = URL_FORMAT + '/configurations?type={2}&tag={3}'
-  response = curl('-u',
+  response = curl(False, '-u',
                   AUTH_FORMAT.format(options.user, options.password),
                   URL_FORMAT.format(options.hostname, options.clustername))
   retcode, errdata = validate_response(response, True)
@@ -519,7 +519,7 @@ def get_config_resp(options, type, error_if_na=True):
 
   if tag != None:
     # Get the config with the tag and return properties
-    response = curl('-u',
+    response = curl(False, '-u',
                     AUTH_FORMAT.format(options.user, options.password),
                     CONFIG_URL_FORMAT.format(options.hostname, options.clustername, type,
tag))
     retcode, errdata = validate_response(response, True)
@@ -638,14 +638,14 @@ def install_services(options):
   SERVICE_URL_FORMAT = URL_FORMAT + '/services?ServiceInfo/state=INIT'
   PUT_IN_INSTALLED = """{"ServiceInfo": {"state": "INSTALLED"}}"""
 
-  response = curl('-u',
+  response = curl(options.printonly, '-u',
                   AUTH_FORMAT.format(options.user, options.password),
                   '-X', 'PUT', '-d',
                   PUT_IN_INSTALLED,
                   SERVICE_URL_FORMAT.format(options.hostname, options.clustername))
-  retcode, errdata = validate_response(response, True)
+  retcode, errdata = validate_response(response, not options.printonly)
   if not retcode == 0:
-    raise FatalException(retcode, errdata)
+    raise FatalException(retcode, errdata + "(Services may already be installed.)")
   pass
 
 
@@ -662,17 +662,26 @@ def validate_response(response, expect_body):
   pass
 
 
-def curl(*args):
+def curl(print_only, *args):
   curl_path = '/usr/bin/curl'
   curl_list = [curl_path]
   for arg in args:
     curl_list.append(arg)
+  if print_only:
+    logger.info("Command to be executed: " + ' '.join(curl_list))
+    return ""
+  pass
   logger.info(' '.join(curl_list))
-  curl_result = subprocess.Popen(
+  osStat = subprocess.Popen(
     curl_list,
     stderr=subprocess.PIPE,
-    stdout=subprocess.PIPE).communicate()[0]
-  return curl_result
+    stdout=subprocess.PIPE)
+  out, err = osStat.communicate()
+  if 0 != osStat.returncode:
+    error = "curl call failed. out: " + out + " err: " + err
+    logger.error(error)
+    raise FatalException(osStat.returncode, error)
+  return out
 
 #
 # Main.
@@ -683,7 +692,7 @@ def main():
 
   parser.add_option("-n", "--printonly",
                     action="store_true", dest="printonly", default=False,
-                    help="Prints all the curl commands to be executed")
+                    help="Prints all the curl commands to be executed (only for write/update
actions)")
   parser.add_option("-o", "--log", dest="logfile", default=UPGRADE_LOG_FILE,
                     help="Log file")
 

http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/59076a55/ambari-server/src/test/python/TestUpgradeScript_HDP2.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestUpgradeScript_HDP2.py b/ambari-server/src/test/python/TestUpgradeScript_HDP2.py
index 3010eec..e7b9446 100644
--- a/ambari-server/src/test/python/TestUpgradeScript_HDP2.py
+++ b/ambari-server/src/test/python/TestUpgradeScript_HDP2.py
@@ -93,10 +93,9 @@ class TestUpgradeHDP2Script(TestCase):
     opm = option_parser_mock.return_value
     path_exists_mock.return_value = True
     shutil_copy_mock = MagicMock()
-    options = MagicMock()
+    options = self.get_mock_options()
     args = ["backup-configs"]
     opm.parse_args.return_value = (options, args)
-    options.logfile = "logfile"
 
     def get_config_resp_side_effect(ops, type, error_if_na):
       if type == "global":
@@ -138,23 +137,18 @@ class TestUpgradeHDP2Script(TestCase):
                            backup_file_mock, file_handler_mock):
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     opm = option_parser_mock.return_value
-    options = MagicMock()
+    options = self.get_mock_options()
     args = ["save-mr-mapping"]
     opm.parse_args.return_value = (options, args)
-    options.logfile = "logfile"
-    options.user = "admin"
-    options.password = "admin"
-    options.hostname = "localhost"
-    options.clustername = "c1"
     curl_mock.side_effect = ['"href" : "', '"href" : "', '"href" : "']
     json_loads_mock.return_value = {"host_components": [{"HostRoles": {"host_name": "host1"}}]}
     UpgradeHelper_HDP2.main()
     expected_curl_calls = [
-      call("-u", "admin:admin",
+      call(False, "-u", "admin:admin",
            "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/MAPREDUCE_CLIENT"),
-      call("-u", "admin:admin",
+      call(False, "-u", "admin:admin",
            "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/TASKTRACKER"),
-      call("-u", "admin:admin",
+      call(False, "-u", "admin:admin",
            "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE/components/JOBTRACKER")]
     curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
     self.assertTrue(write_mapping_mock.called)
@@ -177,14 +171,9 @@ class TestUpgradeHDP2Script(TestCase):
                      backup_file_mock, file_handler_mock, read_mapping_mock, get_yn_mock):
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     opm = option_parser_mock.return_value
-    options = MagicMock()
+    options = self.get_mock_options()
     args = ["delete-mr"]
     opm.parse_args.return_value = (options, args)
-    options.logfile = "logfile"
-    options.user = "admin"
-    options.password = "admin"
-    options.hostname = "localhost"
-    options.clustername = "c1"
     curl_mock.return_value = ''
     get_yn_mock.return_value = True
     read_mapping_mock.return_value = {
@@ -193,13 +182,13 @@ class TestUpgradeHDP2Script(TestCase):
       "MAPREDUCE_CLIENT": ["c6401"]}
     UpgradeHelper_HDP2.main()
     expected_curl_calls = [
-      call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
+      call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/TASKTRACKER"),
-      call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
+      call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/TASKTRACKER"),
-      call("-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
+      call(False, "-u", "admin:admin", "-X", "PUT", "-d", """{"HostRoles": {"state": "MAINTENANCE"}}""",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/JOBTRACKER"),
-      call("-u", "admin:admin", "-X", "DELETE",
+      call(False, "-u", "admin:admin", "-X", "DELETE",
            "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE")]
     curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
     pass
@@ -214,14 +203,9 @@ class TestUpgradeHDP2Script(TestCase):
                        backup_file_mock, file_handler_mock, read_mapping_mock):
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     opm = option_parser_mock.return_value
-    options = MagicMock()
+    options = self.get_mock_options()
     args = ["add-yarn-mr2"]
     opm.parse_args.return_value = (options, args)
-    options.logfile = "logfile"
-    options.user = "admin"
-    options.password = "admin"
-    options.hostname = "localhost"
-    options.clustername = "c1"
     curl_mock.return_value = ''
     read_mapping_mock.return_value = {
       "TASKTRACKER": ["c6401", "c6402"],
@@ -229,31 +213,31 @@ class TestUpgradeHDP2Script(TestCase):
       "MAPREDUCE_CLIENT": ["c6403"]}
     UpgradeHelper_HDP2.main()
     expected_curl_calls = [
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/services/YARN"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/HISTORYSERVER"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/services/MAPREDUCE2/components/MAPREDUCE2_CLIENT"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/NODEMANAGER"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/YARN_CLIENT"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/services/YARN/components/RESOURCEMANAGER"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/HISTORYSERVER"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/NODEMANAGER"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6402/host_components/NODEMANAGER"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/YARN_CLIENT"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6403/host_components/MAPREDUCE2_CLIENT"),
-      call("-u", "admin:admin", "-X", "POST",
+      call(False, "-u", "admin:admin", "-X", "POST",
            "http://localhost:8080/api/v1/clusters/c1/hosts/c6401/host_components/RESOURCEMANAGER")]
     curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
     pass
@@ -267,18 +251,13 @@ class TestUpgradeHDP2Script(TestCase):
                             backup_file_mock, file_handler_mock):
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
     opm = option_parser_mock.return_value
-    options = MagicMock()
+    options = self.get_mock_options()
     args = ["install-yarn-mr2"]
     opm.parse_args.return_value = (options, args)
-    options.logfile = "logfile"
-    options.user = "admin"
-    options.password = "admin"
-    options.hostname = "localhost"
-    options.clustername = "c1"
     curl_mock.return_value = '"href" : "'
     UpgradeHelper_HDP2.main()
     expected_curl_calls = [
-      call("-u", "admin:admin", "-X", "PUT", "-d",
+      call(False, "-u", "admin:admin", "-X", "PUT", "-d",
            """{"ServiceInfo": {"state": "INSTALLED"}}""",
            "http://localhost:8080/api/v1/clusters/c1/services?ServiceInfo/state=INIT")]
     curl_mock.assert_has_calls(expected_curl_calls, any_order=True)
@@ -353,22 +332,31 @@ class TestUpgradeHDP2Script(TestCase):
     self.validate_config_replacememt(curl_mock.call_args_list[4], "global")
     pass
 
+  @patch.object(UpgradeHelper_HDP2, "read_mapping")
+  @patch("subprocess.Popen")
+  @patch.object(UpgradeHelper_HDP2, "get_YN_input")
   @patch.object(logging, 'FileHandler')
   @patch.object(UpgradeHelper_HDP2, "backup_file")
   @patch.object(UpgradeHelper_HDP2, 'curl')
   @patch('optparse.OptionParser')
-  def test_get_and_parse_properties(self, option_parser_mock, curl_mock,
+  def test_print_only(self, option_parser_mock, curl_mock,
+                      backup_file_mock, file_handler_mock, get_yn_mock, popen_mock, read_mapping_mock):
+    file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
+    options = self.get_mock_options(True)
+    get_yn_mock.return_value = True
+    read_mapping_mock.return_value = {
+      "TASKTRACKER": ["c6401", "c6402"]}
+    UpgradeHelper_HDP2.delete_mr(options)
+    self.assertFalse(popen_mock.called)
+    pass
+
+  @patch.object(logging, 'FileHandler')
+  @patch.object(UpgradeHelper_HDP2, "backup_file")
+  @patch.object(UpgradeHelper_HDP2, 'curl')
+  def test_get_and_parse_properties(self, curl_mock,
                                     backup_file_mock, file_handler_mock):
     file_handler_mock.return_value = logging.FileHandler('') # disable creating real file
-    opm = option_parser_mock.return_value
-    options = MagicMock()
-    args = ["update-configs"]
-    opm.parse_args.return_value = (options, args)
-    options.logfile = "logfile"
-    options.user = "admin"
-    options.password = "admin"
-    options.hostname = "localhost"
-    options.clustername = "c1"
+    options = self.get_mock_options()
     curl_mock.side_effect = [
       """{ "href" : "http://localhost:8080/api/v1/clusters/c1",
            "Clusters" : {
@@ -420,32 +408,39 @@ class TestUpgradeHDP2Script(TestCase):
 
   def validate_update_config_call(self, call, type):
     args, kargs = call
-    self.assertTrue(args[3] == 'PUT')
-    self.assertTrue(type in args[5])
+    self.assertTrue(args[4] == 'PUT')
+    self.assertTrue(type in args[6])
     pass
 
   def validate_config_replacememt(self, call, type):
     args, kargs = call
-    self.assertFalse("REPLACE_WITH_" in args[5])
-    self.assertFalse("REPLACE_JH_HOST" in args[5])
-    self.assertFalse("REPLACE_RM_HOST" in args[5])
+    self.assertFalse("REPLACE_WITH_" in args[6])
+    self.assertFalse("REPLACE_JH_HOST" in args[6])
+    self.assertFalse("REPLACE_RM_HOST" in args[6])
     if type == "yarn-site":
-      self.assertTrue("c6401" in args[5])
-      self.assertFalse("an_old_value" in args[5])
+      self.assertTrue("c6401" in args[6])
+      self.assertFalse("an_old_value" in args[6])
     elif type == "mapred-site":
-      self.assertTrue("an_old_value" in args[5])
+      self.assertTrue("an_old_value" in args[6])
     elif type == "global":
-      self.assertTrue("global11" in args[5])
-      self.assertTrue("an_old_value" in args[5])
-      self.assertTrue("mapred.hosts.exclude" in args[5])
+      self.assertTrue("global11" in args[6])
+      self.assertTrue("an_old_value" in args[6])
+      self.assertTrue("mapred.hosts.exclude" in args[6])
     elif (type == "core-site") or (type == "hdfs-site"):
-      self.assertTrue("global11" in args[5])
-      self.assertFalse("an_old_value" in args[5])
-      self.assertFalse("mapred.hosts.exclude" in args[5])
+      self.assertTrue("global11" in args[6])
+      self.assertFalse("an_old_value" in args[6])
+      self.assertFalse("mapred.hosts.exclude" in args[6])
     pass
 
-  def test_read_config(self):
-    pass
+  def get_mock_options(self, printonly=False):
+    options = MagicMock()
+    options.logfile = "logfile"
+    options.user = "admin"
+    options.password = "admin"
+    options.hostname = "localhost"
+    options.clustername = "c1"
+    options.printonly = printonly
+    return options
 
 
 if __name__ == "__main__":


Mime
View raw message