Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 4A874200B35 for ; Tue, 21 Jun 2016 02:38:28 +0200 (CEST) Received: by cust-asf.ponee.io (Postfix) id 490E5160A65; Tue, 21 Jun 2016 00:38:28 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id E8440160A55 for ; Tue, 21 Jun 2016 02:38:26 +0200 (CEST) Received: (qmail 26616 invoked by uid 500); 21 Jun 2016 00:38:26 -0000 Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@ambari.apache.org Delivered-To: mailing list commits@ambari.apache.org Received: (qmail 26606 invoked by uid 99); 21 Jun 2016 00:38:26 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 21 Jun 2016 00:38:26 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 008D5DFC6F; Tue, 21 Jun 2016 00:38:25 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: swapan@apache.org To: commits@ambari.apache.org Message-Id: <40ce44be87a1448f97b1b8b3e5109d92@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: ambari git commit: AMBARI-17306. Filter out MOTD logging from 'llapstatus' command in order to get the output which is JSON parsable. Date: Tue, 21 Jun 2016 00:38:26 +0000 (UTC) archived-at: Tue, 21 Jun 2016 00:38:28 -0000 Repository: ambari Updated Branches: refs/heads/branch-2.4 14e469a12 -> ed5ff0833 AMBARI-17306. Filter out MOTD logging from 'llapstatus' command in order to get the output which is JSON parsable. Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ed5ff083 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ed5ff083 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ed5ff083 Branch: refs/heads/branch-2.4 Commit: ed5ff0833fe998f0c0d9fe2b1eb8859db88becf1 Parents: 14e469a Author: Swapan Shridhar Authored: Mon Jun 20 17:37:45 2016 -0700 Committer: Swapan Shridhar Committed: Mon Jun 20 17:37:45 2016 -0700 ---------------------------------------------------------------------- ambari-server/pom.xml | 1 + .../package/alerts/alert_llap_app_status.py | 78 +++++++- .../package/scripts/hive_server_interactive.py | 80 +++++++- .../0.12.0.2.0/package/scripts/params_linux.py | 2 - .../configuration/hive-interactive-site.xml | 4 +- .../stacks/2.5/HIVE/appComplete_withMOTDmsg.txt | 12 ++ .../stacks/2.5/HIVE/invalidApp_withMOTDmsg.txt | 5 + .../2.5/HIVE/oneContainerDown_withMOTDmsg.txt | 35 ++++ .../stacks/2.5/HIVE/running_withMOTDmsg.txt | 43 +++++ .../stacks/2.5/HIVE/starting_withMOTDmsg.txt | 18 ++ .../stacks/2.5/HIVE/test_hive_server_int.py | 189 ++++++++++++++++++- 11 files changed, 455 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml index d822ac2..6ad7c68 100644 --- a/ambari-server/pom.xml +++ b/ambari-server/pom.xml @@ -269,6 +269,7 @@ src/test/resources/temporal_ganglia_data.txt src/test/resources/users.ldif src/test/resources/mpacks_replay.log + src/test/python/stacks/2.5/HIVE/*.txt src/main/resources/hive-schema-0.10.0.oracle.sql src/main/resources/hive-schema-0.12.0.oracle.sql src/main/resources/db/serial http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_llap_app_status.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_llap_app_status.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_llap_app_status.py index d65322d..33bf139 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_llap_app_status.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/alerts/alert_llap_app_status.py @@ -31,6 +31,7 @@ from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from resource_management.core import shell from resource_management.core.resources import Execute from resource_management.core import global_lock +from resource_management.core.exceptions import Fail OK_MESSAGE = "The application reported a '{0}' state in {1:.3f}s" @@ -164,7 +165,8 @@ def execute(configurations={}, parameters={}, host_name=None): code, output, error = shell.checked_call(llap_status_cmd, user=hive_user, stderr=subprocess.PIPE, timeout=check_command_timeout, logoutput=False) - llap_app_info = json.loads(output) + # Call for getting JSON + llap_app_info = make_valid_json(output) if llap_app_info is None or 'state' not in llap_app_info: alert_label = traceback.format_exc() @@ -219,4 +221,76 @@ def execute(configurations={}, parameters={}, host_name=None): alert_label = traceback.format_exc() traceback.format_exc() result_code = UKNOWN_STATUS_CODE - return (result_code, [alert_label]) \ No newline at end of file + return (result_code, [alert_label]) + + +""" +Remove extra lines from 'llapstatus' status output (eg: because of MOTD logging) so as to have a valid JSON data to be passed in +to JSON converter. +""" +def make_valid_json(output): + ''' + + Note: It is assumed right now that extra lines will be only at the start and not at the end. + + Sample expected JSON to be passed for 'loads' is either of the form : + + Case 'A': + { + "amInfo" : { + "appName" : "llap0", + "appType" : "org-apache-slider", + "appId" : "APP1", + "containerId" : "container_1466036628595_0010_01_000001", + "hostname" : "hostName", + "amWebUrl" : "http://hostName:port/" + }, + "state" : "LAUNCHING", + .... + "desiredInstances" : 1, + "liveInstances" : 0, + .... + .... + } + + or + + Case 'B': + { + "state" : "APP_NOT_FOUND" + } + + ''' + splits = output.split("\n") + + len_splits = len(splits) + if (len_splits < 3): + raise Fail("Malformed JSON data received from 'llapstatus' command. Exiting ....") + + marker_idx = None # To detect where from to start reading for JSON data + for idx, split in enumerate(splits): + curr_elem = split.strip() + if idx + 2 > len_splits: + raise Fail( + "Iterated over the received 'llapstatus' comamnd. Couldn't validate the received output for JSON parsing.") + next_elem = (splits[(idx + 1)]).strip() + if curr_elem == "{": + if next_elem == "\"amInfo\" : {" and (splits[len_splits - 1]).strip() == '}': + # For Case 'A' + marker_idx = idx + break; + elif idx + 3 == len_splits and next_elem.startswith('"state" : ') and (splits[idx + 2]).strip() == '}': + # For Case 'B' + marker_idx = idx + break; + + + # Remove extra logging from possible JSON output + if marker_idx is None: + raise Fail("Couldn't validate the received output for JSON parsing.") + else: + del splits[0:marker_idx] + + scanned_output = '\n'.join(splits) + llap_app_info = json.loads(scanned_output) + return llap_app_info http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py index c055cf1..d1c8401 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py @@ -54,6 +54,7 @@ from hive_service_interactive import hive_service_interactive from hive_interactive import hive_interactive from hive_server import HiveServerDefault +import traceback class HiveServerInteractive(Script): pass @@ -195,6 +196,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive): import params env.set_params(params) Logger.info("Starting LLAP") + LLAP_PACKAGE_CREATION_PATH = Script.get_tmp_dir() LLAP_APP_NAME = 'llap0' unique_name = "llap-slider%s" % datetime.utcnow().strftime('%Y-%m-%d_%H-%M-%S') @@ -202,7 +204,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive): cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llap --instances {params.num_llap_nodes}" " --slider-am-container-mb {params.slider_am_container_mb} --size {params.llap_daemon_container_size}m " " --cache {params.hive_llap_io_mem_size}m --xmx {params.llap_heap_size}m --loglevel {params.llap_log_level}" - " --output {unique_name}") + " --output {LLAP_PACKAGE_CREATION_PATH}/{unique_name}") if params.security_enabled: llap_keytab_splits = params.hive_llap_keytab_file.split("/") Logger.debug("llap_keytab_splits : {0}".format(llap_keytab_splits)) @@ -295,7 +297,80 @@ class HiveServerInteractiveDefault(HiveServerInteractive): llap_status_cmd = format("{stack_root}/current/hive-server2-hive2/bin/hive --service llapstatus --name {app_name} -findAppTimeout {LLAP_APP_STATUS_CMD_TIMEOUT}") code, output, error = shell.checked_call(llap_status_cmd, user=status_params.hive_user, stderr=subprocess.PIPE, logoutput=False) - llap_app_info = json.loads(output) + Logger.info("Received 'llapstatus' command 'output' : {0}".format(output)) + return self._make_valid_json(output) + + + """ + Remove extra lines from 'llapstatus' status output (eg: because of MOTD logging) so as to have a valid JSON data to be passed in + to JSON converter. + """ + def _make_valid_json(self, output): + ''' + + Note: It is assumed right now that extra lines will be only at the start and not at the end. + + Sample expected JSON to be passed for 'loads' is either of the form : + + Case 'A': + { + "amInfo" : { + "appName" : "llap0", + "appType" : "org-apache-slider", + "appId" : "APP1", + "containerId" : "container_1466036628595_0010_01_000001", + "hostname" : "hostName", + "amWebUrl" : "http://hostName:port/" + }, + "state" : "LAUNCHING", + .... + "desiredInstances" : 1, + "liveInstances" : 0, + .... + .... + } + + or + + Case 'B': + { + "state" : "APP_NOT_FOUND" + } + + ''' + splits = output.split("\n") + + len_splits = len(splits) + if (len_splits < 3): + raise Fail ("Malformed JSON data received from 'llapstatus' command. Exiting ....") + + marker_idx = None # To detect where from to start reading for JSON data + for idx, split in enumerate(splits): + curr_elem = split.strip() + if idx+2 > len_splits: + raise Fail("Iterated over the received 'llapstatus' comamnd. Couldn't validate the received output for JSON parsing.") + next_elem = (splits[(idx + 1)]).strip() + if curr_elem == "{": + if next_elem == "\"amInfo\" : {" and (splits[len_splits-1]).strip() == '}': + # For Case 'A' + marker_idx = idx + break; + elif idx+3 == len_splits and next_elem.startswith('"state" : ') and (splits[idx + 2]).strip() == '}': + # For Case 'B' + marker_idx = idx + break; + + Logger.info("Marker index for start of JSON data for 'llapsrtatus' comamnd : {0}".format(marker_idx)) + + # Remove extra logging from possible JSON output + if marker_idx is None: + raise Fail("Couldn't validate the received output for JSON parsing.") + else: + del splits[0:marker_idx] + Logger.info("Removed lines: '1-{0}' from the received 'llapstatus' output to make it valid for JSON parsing.".format(marker_idx)) + + scanned_output = '\n'.join(splits) + llap_app_info = json.loads(scanned_output) return llap_app_info @@ -382,6 +457,7 @@ class HiveServerInteractiveDefault(HiveServerInteractive): except Exception, e: Logger.info("LLAP app '{0}' did not come up after a wait of {1} seconds.".format(llap_app_name, time.time() - curr_time)) + traceback.print_exc() return False def get_log_folder(self): http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py index 9dbc18d..6a53dca 100644 --- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py +++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py @@ -375,8 +375,6 @@ jars_in_hive_lib = format("{hive_lib}/*.jar") start_hiveserver2_path = format("{tmp_dir}/start_hiveserver2_script") start_metastore_path = format("{tmp_dir}/start_metastore_script") -yarn_scheduler_allocation_minimum_mb = config['configurations']['yarn-site']['properties']['yarn.scheduler.minimum-allocation-mb'] - hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] if 'role' in config and config['role'] in ["HIVE_SERVER", "HIVE_METASTORE"]: http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml index 9c7e865..a0076b3 100644 --- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml +++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/HIVE/configuration/hive-interactive-site.xml @@ -448,7 +448,7 @@ limitations under the License. hive.llap.daemon.yarn.container.mb

Memory per daemon

Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution. - {{yarn_scheduler_allocation_minimum_mb}} + 341

int MB @@ -606,7 +606,7 @@ limitations under the License. --> Memory per daemon Total memory used by individual LLAP daemons. This includes memory for the cache as well as for the query execution. - {{yarn_scheduler_allocation_minimum_mb}} + 341

int MB http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/test/python/stacks/2.5/HIVE/appComplete_withMOTDmsg.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/appComplete_withMOTDmsg.txt b/ambari-server/src/test/python/stacks/2.5/HIVE/appComplete_withMOTDmsg.txt new file mode 100644 index 0000000..34b8d67 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/appComplete_withMOTDmsg.txt @@ -0,0 +1,12 @@ +######## Hortonworks ############# +This is MOTD message, added for testing in qe infra +{ + "amInfo" : { + "appName" : "llap", + "appType" : "org-apache-slider", + "appId" : "application_1455662455106_10882" + }, + "state" : "COMPLETE", + "appStartTime" : 1459625790218, + "appFinishTime" : 1459625939033 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/test/python/stacks/2.5/HIVE/invalidApp_withMOTDmsg.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/invalidApp_withMOTDmsg.txt b/ambari-server/src/test/python/stacks/2.5/HIVE/invalidApp_withMOTDmsg.txt new file mode 100644 index 0000000..f7d2832 --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/invalidApp_withMOTDmsg.txt @@ -0,0 +1,5 @@ +######## Hortonworks ############# +This is MOTD message, added for testing in qe infra +{ + "state" : "APP_NOT_FOUND" +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/test/python/stacks/2.5/HIVE/oneContainerDown_withMOTDmsg.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/oneContainerDown_withMOTDmsg.txt b/ambari-server/src/test/python/stacks/2.5/HIVE/oneContainerDown_withMOTDmsg.txt new file mode 100644 index 0000000..84b331a --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/oneContainerDown_withMOTDmsg.txt @@ -0,0 +1,35 @@ +######## Hortonworks ############# +This is MOTD message, added for testing in qe infra +{ + "amInfo" : { + "appName" : "llap", + "appType" : "org-apache-slider", + "appId" : "application_1455662455106_10882", + "containerId" : "container_e14_1455662455106_10882_01_000001", + "hostname" : "HOST_REPLACED", + "amWebUrl" : "http://HOST_REPLACED:1025/" + }, + "state" : "RUNNING_PARTIAL", + "originalConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/snapshot", + "generatedConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/generated", + "desiredInstances" : 3, + "liveInstances" : 2, + "appStartTime" : 1459625802169, + "llapInstances" : [ { + "hostname" : "HOST_REPLACED", + "containerId" : "container_e14_1455662455106_10882_01_000003", + "statusUrl" : "http://HOST_REPLACED:15002/status", + "webUrl" : "http://HOST_REPLACED:15002", + "rpcPort" : 15001, + "mgmtPort" : 15004, + "shufflePort" : 15551 + }, { + "hostname" : "HOST_REPLACED", + "containerId" : "container_e14_1455662455106_10882_01_000002", + "statusUrl" : "http://HOST_REPLACED:15002/status", + "webUrl" : "http://HOST_REPLACED:15002", + "rpcPort" : 15001, + "mgmtPort" : 15004, + "shufflePort" : 15551 + } ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg.txt b/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg.txt new file mode 100644 index 0000000..d5a4b8c --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg.txt @@ -0,0 +1,43 @@ +######## Hortonworks ############# +This is MOTD message, added for testing in qe infra +{ + "amInfo" : { + "appName" : "llap", + "appType" : "org-apache-slider", + "appId" : "application_1455662455106_10882", + "containerId" : "container_e14_1455662455106_10882_01_000001", + "hostname" : "HOST_REPLACED", + "amWebUrl" : "http://HOST_REPLACED:1025/" + }, + "state" : "RUNNING_ALL", + "originalConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/snapshot", + "generatedConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/generated", + "desiredInstances" : 3, + "liveInstances" : 3, + "appStartTime" : 1459625802169, + "llapInstances" : [ { + "hostname" : "HOST_REPLACED", + "containerId" : "container_e14_1455662455106_10882_01_000003", + "statusUrl" : "http://HOST_REPLACED:15002/status", + "webUrl" : "http://HOST_REPLACED:15002", + "rpcPort" : 15001, + "mgmtPort" : 15004, + "shufflePort" : 15551 + }, { + "hostname" : "HOST_REPLACED", + "containerId" : "container_e14_1455662455106_10882_01_000002", + "statusUrl" : "http://HOST_REPLACED:15002/status", + "webUrl" : "http://HOST_REPLACED:15002", + "rpcPort" : 15001, + "mgmtPort" : 15004, + "shufflePort" : 15551 + }, { + "hostname" : "HOST_REPLACED", + "containerId" : "container_e14_1455662455106_10882_01_000004", + "statusUrl" : "http://HOST_REPLACED:15002/status", + "webUrl" : "http://HOST_REPLACED:15002", + "rpcPort" : 15001, + "mgmtPort" : 15004, + "shufflePort" : 15551 + } ] +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/test/python/stacks/2.5/HIVE/starting_withMOTDmsg.txt ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/starting_withMOTDmsg.txt b/ambari-server/src/test/python/stacks/2.5/HIVE/starting_withMOTDmsg.txt new file mode 100644 index 0000000..507217f --- /dev/null +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/starting_withMOTDmsg.txt @@ -0,0 +1,18 @@ +######## Hortonworks ############# +This is MOTD message, added for testing in qe infra +{ + "amInfo" : { + "appName" : "llap", + "appType" : "org-apache-slider", + "appId" : "application_1455662455106_10882", + "containerId" : "container_e14_1455662455106_10882_01_000001", + "hostname" : "HOST_REPLACED", + "amWebUrl" : "http://HOST_REPLACED:1025/" + }, + "state" : "LAUNCHING", + "originalConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/snapshot", + "generatedConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/generated", + "desiredInstances" : 3, + "liveInstances" : 0, + "appStartTime" : 1459625802169 +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/ed5ff083/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py index 348a17d..1e1850e 100644 --- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py +++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py @@ -28,7 +28,10 @@ from resource_management.libraries import functions from resource_management.core.logger import Logger from resource_management.libraries.script.config_dictionary import UnknownConfiguration from hive_server_interactive import HiveServerInteractiveDefault +from resource_management.libraries.script.script import Script +from resource_management.core import shell +@patch("resource_management.libraries.Script.get_tmp_dir", new=MagicMock(return_value=('/var/lib/ambari-agent/tmp'))) @patch.object(functions, "get_stack_version", new=MagicMock(return_value="2.0.0.0-1234")) @patch("resource_management.libraries.functions.check_thrift_port_sasl", new=MagicMock()) @patch("resource_management.libraries.functions.get_user_call_output.get_user_call_output", @@ -72,15 +75,17 @@ class TestHiveServerInteractive(RMFTestCase): Tests HSI start with llap package creation output having single line. Sample output : "Prepared llap-slider-05Apr2016/run.sh for running LLAP" """ + #@patch("Script.get_tmp_dir()") @patch("os.path.isfile") @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs") @patch("socket.socket") @patch("time.sleep") - def test_start_default_with_llap_single_line_output(self, sleep_mock, socket_mock, copy_to_hfds_mock, is_file_mock): + def test_start_default_with_llap_single_line_output(self, sleep_mock, socket_mock, copy_to_hfds_mock, is_file_mock): #, get_tmp_dir_mock): self.maxDiff = None copy_to_hfds_mock.return_value = False s = socket_mock.return_value is_file_mock.return_value = True + #get_tmp_dir_mock.return_value = "/var/lib/ambari-agent/tmp" self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hive_server_interactive.py", classname="HiveServerInteractive", command="start", @@ -88,7 +93,13 @@ class TestHiveServerInteractive(RMFTestCase): stack_version=self.STACK_VERSION, target=RMFTestCase.TARGET_COMMON_SERVICES, checked_call_mocks=[(0, "Prepared llap-slider-05Apr2016/run.sh for running LLAP", ""), - (0, "{\"state\":\"RUNNING_ALL\"}", ""), (0, "OK.", "")], + (0, """{ + \"state\" : \"RUNNING_ALL\" + }""", ""), + (0, """{ + \"state\" : \"RUNNING_ALL\" + }""", ""), + (0, "OK.", "")], ) self.assert_configure_default() @@ -141,7 +152,12 @@ class TestHiveServerInteractive(RMFTestCase): checked_call_mocks=[(0, "UNWANTED_STRING \n " " Prepared llap-slider-05Apr2016/run.sh for running LLAP \n " "UNWANTED_STRING \n ", ""), - (0, "{\"state\":\"RUNNING_ALL\"}", ""), (0, "OK.", "")], + (0, """{ + \"state\" : \"RUNNING_ALL\" + }""", ""), + (0, """{ + \"state\" : \"RUNNING_ALL\" + }""", ""), (0, "OK.", "")], ) self.assert_configure_default() @@ -384,7 +400,172 @@ class TestHiveServerInteractive(RMFTestCase): - # llap app 'status check' related tests + + + + + # Tests for function '_make_valid_json()' with will be passed in with 'llapstatus' output which may be : + # (1). A string parseable as JSON, or + # (2). May have extra lines in beginning (eg: from MOTD logging embedded), which needs to be removed before parsed as JSON + + # Status : RUNNING having MOTD lines in beginning + def test_make_valid_json_1(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/running_withMOTDmsg.txt","r") + llap_app_info = input_file_handle.read() + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Set up expected output + expected_ouput_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/running.json","r") + expected_ouput_data = expected_ouput_file_handle.read() + expected_ouput_data_as_json = json.loads(expected_ouput_data) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_ouput_data_as_json) + + # Status : RUNNING w/o MOTD lines in beginning + # Expected : No change + def test_make_valid_json_2(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/running.json","r") + llap_app_info = input_file_handle.read() + expected_llap_app_info_as_json = json.loads(llap_app_info) + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_llap_app_info_as_json) + + + + # Status : RUNNING_PARTIAL (2 out of 3 running -> < 80% instances ON) having MOTD lines in beginning + def test_make_valid_json_3(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/oneContainerDown_withMOTDmsg.txt","r") + llap_app_info = input_file_handle.read() + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Set up expected output + expected_ouput_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/oneContainerDown.json","r") + expected_ouput_data = expected_ouput_file_handle.read() + expected_ouput_data_as_json = json.loads(expected_ouput_data) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_ouput_data_as_json) + + # Status : RUNNING_PARTIAL (2 out of 3 running -> < 80% instances ON) w/o MOTD lines in beginning + # Expected : No change + def test_make_valid_json_4(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/oneContainerDown.json","r") + llap_app_info = input_file_handle.read() + expected_llap_app_info_as_json = json.loads(llap_app_info) + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_llap_app_info_as_json) + + + + # Status : LAUNCHING having MOTD lines in beginning + def test_make_valid_json_5(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/starting_withMOTDmsg.txt","r") + llap_app_info = input_file_handle.read() + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Set up expected output + expected_ouput_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/starting.json","r") + expected_ouput_data = expected_ouput_file_handle.read() + expected_ouput_data_as_json = json.loads(expected_ouput_data) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_ouput_data_as_json) + + # Status : LAUNCHING w/o MOTD lines in beginning + # Expected : No change + def test_make_valid_json_6(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/starting.json","r") + llap_app_info = input_file_handle.read() + expected_llap_app_info_as_json = json.loads(llap_app_info) + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_llap_app_info_as_json) + + + + # Status : COMPLETE having MOTD lines in beginning + def test_make_valid_json_7(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/appComplete_withMOTDmsg.txt","r") + llap_app_info = input_file_handle.read() + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Set up expected output + expected_ouput_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/appComplete.json","r") + expected_ouput_data = expected_ouput_file_handle.read() + expected_ouput_data_as_json = json.loads(expected_ouput_data) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_ouput_data_as_json) + + # Status : COMPLETE w/o MOTD lines in beginning + # Expected : No change + def test_make_valid_json_8(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/appComplete.json","r") + llap_app_info = input_file_handle.read() + expected_llap_app_info_as_json = json.loads(llap_app_info) + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_llap_app_info_as_json) + + + + # Status : INVALID APP having MOTD lines in beginning + def test_make_valid_json_9(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/invalidApp_withMOTDmsg.txt","r") + llap_app_info = input_file_handle.read() + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Set up expected output + expected_ouput_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/invalidApp.json","r") + expected_ouput_data = expected_ouput_file_handle.read() + expected_ouput_data_as_json = json.loads(expected_ouput_data) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_ouput_data_as_json) + + # Status : INVALID APP w/o MOTD lines in beginning + # Expected : No change + def test_make_valid_json_10(self): + # Setting up input for fn. '_make_valid_json()' + input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/invalidApp.json","r") + llap_app_info = input_file_handle.read() + expected_llap_app_info_as_json = json.loads(llap_app_info) + + llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info) + + # Verification + self.assertEqual(llap_app_info_as_json, expected_llap_app_info_as_json) + + + + + # Tests for fn : 'check_llap_app_status()' + # Status : RUNNING @patch("time.sleep")