ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [28/31] ambari git commit: AMBARI-18943. If multiple HSI components are deployed then HSI start should not restart llap (smohanty)
Date Mon, 21 Nov 2016 16:08:59 GMT
AMBARI-18943. If multiple HSI components are deployed then HSI start should not restart llap
(smohanty)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e1638ce7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e1638ce7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e1638ce7

Branch: refs/heads/branch-feature-AMBARI-18456
Commit: e1638ce7dfa9edd9ae025cd3d8ca37795dc6cd46
Parents: 3b0c35b
Author: Sumit Mohanty <smohanty@hortonworks.com>
Authored: Sun Nov 20 15:26:38 2016 -0800
Committer: Sumit Mohanty <smohanty@hortonworks.com>
Committed: Sun Nov 20 15:30:13 2016 -0800

----------------------------------------------------------------------
 .../package/scripts/hive_server_interactive.py  |   22 +-
 .../0.12.0.2.0/package/scripts/params_linux.py  |    1 +
 .../stacks/2.5/HIVE/test_hive_server_int.py     |  112 ++
 .../test/python/stacks/2.5/configs/hsi_ha.json  | 1254 ++++++++++++++++++
 4 files changed, 1387 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/e1638ce7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 0e00c3a..d3773a7 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -252,9 +252,24 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
     def _llap_start(self, env, cleanup=False):
       import params
       env.set_params(params)
+
+      LLAP_APP_NAME = 'llap0'
+
+      if params.hive_server_interactive_ha:
+        """
+        Check llap app state
+        """
+        Logger.info("HSI HA is enabled. Checking if LLAP is already running ...")
+        status = self.check_llap_app_status(LLAP_APP_NAME, 2, params.hive_server_interactive_ha)
+        if status:
+          Logger.info("LLAP app '{0}' is already running.".format(LLAP_APP_NAME))
+          return True
+        else:
+          Logger.info("LLAP app '{0}' is not running. llap will be started.".format(LLAP_APP_NAME))
+        pass
+
       Logger.info("Starting LLAP")
       LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir()
-      LLAP_APP_NAME = 'llap0'
 
       unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S')
 
@@ -446,7 +461,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
     Parameters: llap_app_name : deployed llap app name.
                 num_retries :   Number of retries to check the LLAP app status.
     """
-    def check_llap_app_status(self, llap_app_name, num_retries):
+    def check_llap_app_status(self, llap_app_name, num_retries, return_immediately_if_stopped=False):
       # counters based on various states.
       curr_time = time.time()
 
@@ -465,6 +480,9 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
           Logger.error("Malformed JSON data received for LLAP app. Exiting ....")
           return False
 
+        if return_immediately_if_stopped and (llap_app_info['state'].upper() in ('APP_NOT_FOUND',
'COMPLETE')):
+          return False
+
         if llap_app_info['state'].upper() == 'RUNNING_ALL':
           Logger.info(
             "LLAP app '{0}' in '{1}' state.".format(llap_app_name, llap_app_info['state']))

http://git-wip-us.apache.org/repos/asf/ambari/blob/e1638ce7/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index e9372f9..30fc82a 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -312,6 +312,7 @@ hive_server_host = hive_server_hosts[0] if len(hive_server_hosts) >
0 else None
 
 hive_server_interactive_hosts = default('/clusterHostInfo/hive_server_interactive_hosts',
[])
 hive_server_interactive_host = hive_server_interactive_hosts[0] if len(hive_server_interactive_hosts)
> 0 else None
+hive_server_interactive_ha = True if len(hive_server_interactive_hosts) > 1 else False
 # End, Common Hosts and Ports
 
 hive_transport_mode = config['configurations']['hive-site']['hive.server2.transport.mode']

http://git-wip-us.apache.org/repos/asf/ambari/blob/e1638ce7/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
index 7477457..3221d5d 100644
--- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
+++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
@@ -131,6 +131,118 @@ class TestHiveServerInteractive(RMFTestCase):
     self.assertNoMoreResources()
 
   """
+  Tests HSI start with llap package creation output having single line.
+  Sample output : "Prepared llap-slider-05Apr2016/run.sh for running LLAP"
+  HSI HA is enabled and llap is stopped, so it will be restartd
+  """
+  #@patch("Script.get_tmp_dir()")
+  @patch("os.path.isfile")
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+  @patch("socket.socket")
+  @patch("time.sleep")
+  def test_start_hsi_ha_with_llap_single_line_output(self, sleep_mock, socket_mock, copy_to_hfds_mock,
is_file_mock): #, get_tmp_dir_mock):
+    self.maxDiff = None
+    copy_to_hfds_mock.return_value = False
+    s = socket_mock.return_value
+    is_file_mock.return_value = True
+    #get_tmp_dir_mock.return_value = "/var/lib/ambari-agent/tmp"
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server_interactive.py",
+                       classname="HiveServerInteractive",
+                       command="start",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.5/configs/hsi_ha.json",
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES,
+                       checked_call_mocks=[(0, """{
+                                                      \"state\" : \"COMPLETED\"
+                                                   }""", ""),
+                                           (0, "Prepared llap-slider-05Apr2016/run.sh for
running LLAP", ""),
+                                           (0, """{
+                                                      \"state\" : \"RUNNING_ALL\"
+                                                   }""", ""),
+                                           (0, """{
+                                                      \"state\" : \"RUNNING_ALL\"
+                                                   }""", ""),
+                                           (0, "OK.", "")],
+                       )
+
+    self.assert_configure_default()
+
+    self.assertResourceCalled('Execute',
+                              '/home/hive/llap-slider-05Apr2016/run.sh',
+                              logoutput= True, user='hive'
+    )
+    self.assertResourceCalled('Execute',
+                              'hive --config /usr/hdp/current/hive-server2-hive2/conf/conf.server
--service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020 OK.',
+                              environment={'PATH': '/usr/hdp/current/hadoop-client/bin'},
+                              user='hive'
+    )
+    self.assertResourceCalled('Execute',
+                              '/tmp/start_hiveserver2_interactive_script /var/run/hive/hive-server2-interactive.out
/var/log/hive/hive-server2-interactive.err /var/run/hive/hive-interactive.pid /usr/hdp/current/hive-server2-hive2/conf/conf.server
/var/log/hive',
+                              environment={'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
+                                           'HIVE_BIN': 'hive2',
+                                           'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+                              not_if="ls /var/run/hive/hive-interactive.pid >/dev/null
2>&1 && ps -p 123 >/dev/null 2>&1",
+                              user='hive',
+                              path=['/bin:/usr/hdp/current/hive-server2-hive2/bin:/usr/hdp/current/hadoop-client/bin'],
+                              )
+    self.assertResourceCalled('Execute',
+                              '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar
org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\'
hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
+                              path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
+                              tries=5,
+                              try_sleep=10
+    )
+    self.assertNoMoreResources()
+
+  """
+  Tests HSI start with llap running and HA deployment. As llap is running it will not be
started."
+  """
+  #@patch("Script.get_tmp_dir()")
+  @patch("os.path.isfile")
+  @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+  @patch("socket.socket")
+  @patch("time.sleep")
+  def test_start_with_ha_and_llap_running(self, sleep_mock, socket_mock, copy_to_hfds_mock,
is_file_mock): #, get_tmp_dir_mock):
+    self.maxDiff = None
+    copy_to_hfds_mock.return_value = False
+    s = socket_mock.return_value
+    is_file_mock.return_value = True
+    #get_tmp_dir_mock.return_value = "/var/lib/ambari-agent/tmp"
+    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server_interactive.py",
+                       classname="HiveServerInteractive",
+                       command="start",
+                       config_file=self.get_src_folder() + "/test/python/stacks/2.5/configs/hsi_ha.json",
+                       stack_version=self.STACK_VERSION,
+                       target=RMFTestCase.TARGET_COMMON_SERVICES,
+                       checked_call_mocks=[(0, """{
+                                                      \"state\" : \"RUNNING_ALL\"
+                                                   }""", "")],
+                       )
+
+    self.assert_configure_default()
+
+    self.assertResourceCalled('Execute',
+                              'hive --config /usr/hdp/current/hive-server2-hive2/conf/conf.server
--service metatool -updateLocation hdfs://c6401.ambari.apache.org:8020 OK.',
+                              environment={'PATH': '/usr/hdp/current/hadoop-client/bin'},
+                              user='hive'
+    )
+    self.assertResourceCalled('Execute',
+                              '/tmp/start_hiveserver2_interactive_script /var/run/hive/hive-server2-interactive.out
/var/log/hive/hive-server2-interactive.err /var/run/hive/hive-interactive.pid /usr/hdp/current/hive-server2-hive2/conf/conf.server
/var/log/hive',
+                              environment={'HADOOP_HOME': '/usr/hdp/current/hadoop-client',
+                                           'HIVE_BIN': 'hive2',
+                                           'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+                              not_if="ls /var/run/hive/hive-interactive.pid >/dev/null
2>&1 && ps -p 123 >/dev/null 2>&1",
+                              user='hive',
+                              path=['/bin:/usr/hdp/current/hive-server2-hive2/bin:/usr/hdp/current/hadoop-client/bin'],
+                              )
+    self.assertResourceCalled('Execute',
+                              '/usr/jdk64/jdk1.7.0_45/bin/java -cp /usr/lib/ambari-agent/DBConnectionVerification.jar:/usr/hdp/current/hive-server2-hive2/lib/mysql-connector-java.jar
org.apache.ambari.server.DBConnectionVerification \'jdbc:mysql://c6402.ambari.apache.org/hive?createDatabaseIfNotExist=true\'
hive \'!`"\'"\'"\' 1\' com.mysql.jdbc.Driver',
+                              path=['/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'],
+                              tries=5,
+                              try_sleep=10
+    )
+    self.assertNoMoreResources()
+
+  """
   Tests HSI start with llap package creation output having multiple lines.
   Sample output : "UNWANTED STRING \n Prepared llap-slider-05Apr2016/run.sh for running LLAP
\n UNWANTED STRING \n"
   """


Mime
View raw message