ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [06/16] ambari git commit: AMBARI-22498. Remove trailing lines (if any) from llapstatus command before converting the o/p to JSON.
Date Mon, 27 Nov 2017 13:00:01 GMT
AMBARI-22498. Remove trailing lines (if any) from llapstatus command before converting the
o/p to JSON.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/677e27e6
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/677e27e6
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/677e27e6

Branch: refs/heads/branch-feature-AMBARI-22457
Commit: 677e27e658b1fe6e272789ce97ecfbaa5ad0d0ca
Parents: 416570d
Author: Swapan Shridhar <sshridhar@hortonworks.com>
Authored: Wed Nov 22 10:24:38 2017 -0800
Committer: Swapan Shridhar <sshridhar@hortonworks.com>
Committed: Wed Nov 22 10:26:53 2017 -0800

----------------------------------------------------------------------
 .../package/scripts/hive_server_interactive.py  | 20 +++++++--
 .../HIVE/running_withMOTDmsg_andTrailingMsg.txt | 46 ++++++++++++++++++++
 .../stacks/2.5/HIVE/test_hive_server_int.py     | 21 +++++++++
 3 files changed, 84 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/677e27e6/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
index 57cbcd0..32322cd 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/hive_server_interactive.py
@@ -416,13 +416,14 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
 
 
     """
-    Remove extra lines from 'llapstatus' status output (eg: because of MOTD logging) so as
to have a valid JSON data to be passed in
-    to JSON converter.
+    Remove extra lines (begginning/end) from 'llapstatus' status output (eg: because of MOTD
logging) so as to have 
+    a valid JSON data to be passed in to JSON converter.
     """
     def _make_valid_json(self, output):
       '''
 
-      Note: It is assumed right now that extra lines will be only at the start and not at
the end.
+      Note: Extra lines (eg: because of MOTD) may be at the start or the end (some other
logging getting appended)
+      of the passed-in data.
 
       Sample expected JSON to be passed for 'loads' is either of the form :
 
@@ -458,6 +459,19 @@ class HiveServerInteractiveDefault(HiveServerInteractive):
       if (len_splits < 3):
         raise Fail ("Malformed JSON data received from 'llapstatus' command. Exiting ....")
 
+      # Firstly, remove extra lines from the END.
+      updated_splits = []
+      for itr, line in enumerate(reversed(splits)):
+        if line == "}": # Our assumption of end of JSON data.
+          updated_splits = splits[:-itr]
+          break
+
+      if len(updated_splits) > 0:
+        splits = updated_splits
+        len_splits = len(splits)
+
+
+      # Secondly, remove extra lines from the BEGGINNING.
       marker_idx = None # To detect where from to start reading for JSON data
       for idx, split in enumerate(splits):
         curr_elem = split.strip()

http://git-wip-us.apache.org/repos/asf/ambari/blob/677e27e6/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt
b/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt
new file mode 100644
index 0000000..394faef
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt
@@ -0,0 +1,46 @@
+######## Hortonworks #############
+This is MOTD message, added for testing in qe infra
+{
+  "amInfo" : {
+    "appName" : "llap",
+    "appType" : "org-apache-slider",
+    "appId" : "application_1455662455106_10882",
+    "containerId" : "container_e14_1455662455106_10882_01_000001",
+    "hostname" : "HOST_REPLACED",
+    "amWebUrl" : "http://HOST_REPLACED:1025/"
+  },
+  "state" : "RUNNING_ALL",
+  "originalConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/snapshot",
+  "generatedConfigurationPath" : "hdfs://HOST_REPLACED:8020/user/USER_REPLACED/.slider/cluster/llap/generated",
+  "desiredInstances" : 3,
+  "liveInstances" : 3,
+  "appStartTime" : 1459625802169,
+  "llapInstances" : [ {
+    "hostname" : "HOST_REPLACED",
+    "containerId" : "container_e14_1455662455106_10882_01_000003",
+    "statusUrl" : "http://HOST_REPLACED:15002/status",
+    "webUrl" : "http://HOST_REPLACED:15002",
+    "rpcPort" : 15001,
+    "mgmtPort" : 15004,
+    "shufflePort" : 15551
+  }, {
+    "hostname" : "HOST_REPLACED",
+    "containerId" : "container_e14_1455662455106_10882_01_000002",
+    "statusUrl" : "http://HOST_REPLACED:15002/status",
+    "webUrl" : "http://HOST_REPLACED:15002",
+    "rpcPort" : 15001,
+    "mgmtPort" : 15004,
+    "shufflePort" : 15551
+  }, {
+    "hostname" : "HOST_REPLACED",
+    "containerId" : "container_e14_1455662455106_10882_01_000004",
+    "statusUrl" : "http://HOST_REPLACED:15002/status",
+    "webUrl" : "http://HOST_REPLACED:15002",
+    "rpcPort" : 15001,
+    "mgmtPort" : 15004,
+    "shufflePort" : 15551
+  } ]
+}
+
+# THIS IS A DUMMY TRAILING MESSAGE 1
+# THIS IS A DUMMY TRAILING MESSAGE 2
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/677e27e6/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
index cf79ec7..4eb16c2 100644
--- a/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
+++ b/ambari-server/src/test/python/stacks/2.5/HIVE/test_hive_server_int.py
@@ -915,6 +915,27 @@ class TestHiveServerInteractive(RMFTestCase):
 
 
 
+  # Tests for function '_make_valid_json()' : will be passed in with 'llapstatus' output
which will be :
+  #     (1). A string parseable as JSON, but has 2 and 3.
+  #     (2). Has extra lines in beginning (eg: from MOTD logging embedded)
+  #          AND/OR
+  #     (3). Extra lines at the end.
+
+  # Begginning and end lines need to be removed before parsed as JSON
+  def test_make_valid_json_11(self):
+      # Setting up input for fn. '_make_valid_json()'
+      input_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/running_withMOTDmsg_andTrailingMsg.txt","r")
+      llap_app_info = input_file_handle.read()
+      llap_app_info_as_json = self.hsi._make_valid_json(llap_app_info)
+
+      # Set up expected output
+      expected_ouput_file_handle = open(self.get_src_folder() + "/test/python/stacks/2.5/HIVE/running.json","r")
+      expected_ouput_data = expected_ouput_file_handle.read()
+      expected_ouput_data_as_json = json.loads(expected_ouput_data)
+
+      # Verification
+      self.assertEqual(llap_app_info_as_json, expected_ouput_data_as_json)
+
 
   # Tests for fn : 'check_llap_app_status_in_hdp_tp()'
 


Mime
View raw message