ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ja...@apache.org
Subject ambari git commit: AMBARI-14816: Run a service check on PXF service for Hive (lavjain via jaoki)
Date Wed, 03 Feb 2016 00:19:51 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 6fb00585c -> 7632b7bb2


AMBARI-14816: Run a service check on PXF service for Hive (lavjain via jaoki)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7632b7bb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7632b7bb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7632b7bb

Branch: refs/heads/trunk
Commit: 7632b7bb2454a67015308da25c588dd0bf4aafe2
Parents: 6fb0058
Author: Jun Aoki <jaoki@apache.org>
Authored: Tue Feb 2 16:19:24 2016 -0800
Committer: Jun Aoki <jaoki@apache.org>
Committed: Tue Feb 2 16:19:24 2016 -0800

----------------------------------------------------------------------
 .../PXF/3.0.0/package/scripts/params.py         |   6 +-
 .../PXF/3.0.0/package/scripts/pxf_constants.py  |   1 +
 .../PXF/3.0.0/package/scripts/service_check.py  | 101 +++++++++++++------
 3 files changed, 76 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/7632b7bb/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
index 36dc7c1..7749de7 100644
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
@@ -16,7 +16,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 """
 
-import os
 import functools
 
 from resource_management import Script
@@ -35,6 +34,7 @@ pxf_group = pxf_user
 hdfs_superuser = config['configurations']['hadoop-env']['hdfs_user']
 hdfs_superuser_group = config["configurations"]["hdfs-site"]["dfs.permissions.superusergroup"]
 user_group = config["configurations"]["cluster-env"]["user_group"]
+hive_user = default('configurations/hive-env/hive_user', None)
 tomcat_group = "tomcat"
 
 # Directories
@@ -54,11 +54,13 @@ realm_name = config['configurations']['kerberos-env']['realm']
 #HBase
 is_hbase_installed = default("/clusterHostInfo/hbase_master_hosts", None) is not None
 
+#Hive
+is_hive_installed = default("/clusterHostInfo/hive_server_host", None) is not None
+
 # HDFS
 hdfs_site = config['configurations']['hdfs-site']
 default_fs = config['configurations']['core-site']['fs.defaultFS']
 
-security_enabled = config['configurations']['cluster-env']['security_enabled']
 hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
 kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
 hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']

http://git-wip-us.apache.org/repos/asf/ambari/blob/7632b7bb/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
index 3138379..9d93a38 100644
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/pxf_constants.py
@@ -24,3 +24,4 @@ pxf_hdfs_test_dir = "/pxf_hdfs_smoke_test"
 pxf_hdfs_read_test_file = pxf_hdfs_test_dir + "/pxf_smoke_test_read_data"
 pxf_hdfs_write_test_file = pxf_hdfs_test_dir + "/pxf_smoke_test_write_data"
 pxf_hbase_test_table = "pxf_hbase_smoke_test_table"
+pxf_hive_test_table = "pxf_hive_smoke_test_table"

http://git-wip-us.apache.org/repos/asf/ambari/blob/7632b7bb/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
index 72b59e4..064be04 100644
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/service_check.py
@@ -24,8 +24,6 @@ from resource_management.core.resources.system import Execute
 from pxf_utils import makeHTTPCall, runLocalCmd
 import pxf_constants
 
-import sys
-
 class PXFServiceCheck(Script):
   """
   Runs a set of simple PXF tests to verify if the service has been setup correctly
@@ -51,10 +49,11 @@ class PXFServiceCheck(Script):
     import params
     self.pxf_version = self.__get_pxf_protocol_version()
     try:
-      self.cleanup_test_data()
       self.run_hdfs_tests()
       if params.is_hbase_installed:
         self.run_hbase_tests()
+      if params.is_hive_installed:
+        self.run_hive_tests()
     except:
       msg = "PXF service check failed"
       Logger.error(msg)
@@ -75,6 +74,8 @@ class PXFServiceCheck(Script):
     self.__cleanup_hdfs_data()
     if params.is_hbase_installed:
       self.__cleanup_hbase_data()
+    if params.is_hive_installed:
+      self.__cleanup_hive_data()
 
 
   def __get_pxf_protocol_version(self):
@@ -98,17 +99,33 @@ class PXFServiceCheck(Script):
     raise Fail(msg)
 
 
+  def __check_pxf_read(self, headers):
+    """
+    Performs a generic PXF read
+    """
+    url = self.base_url + self.pxf_version + "/Fragmenter/getFragments?path="
+    try:
+      response = makeHTTPCall(url, headers)
+      if not "PXFFragments" in response:
+        Logger.error("Unable to find PXFFragments in the response")
+        raise
+    except:
+      msg = "PXF data read failed"
+      raise Fail(msg)
+
+
+  # HDFS Routines
   def run_hdfs_tests(self):
     """
     Runs a set of PXF HDFS checks
     """
     Logger.info("Running PXF HDFS checks")
-    self.__check_if_client_exists("HDFS")
+    self.__check_if_client_exists("Hadoop-HDFS")
+    self.__cleanup_hdfs_data()
     self.__write_hdfs_data()
     self.__check_pxf_hdfs_read()
     self.__check_pxf_hdfs_write()
 
-
   def __write_hdfs_data(self):
     """
     Writes some test HDFS data for the tests
@@ -127,22 +144,6 @@ class PXFServiceCheck(Script):
         action="create_on_execute"
         )
 
- 
-  def __check_pxf_read(self, headers):
-    """
-    Performs a generic PXF read 
-    """
-    url = self.base_url + self.pxf_version + "/Fragmenter/getFragments?path="
-    try:
-      response = makeHTTPCall(url, headers)
-      if not "PXFFragments" in response:
-        Logger.error("Unable to find PXFFragments in the response")
-        raise 
-    except:
-      msg = "PXF data read failed"
-      raise Fail(msg)
-
-
   def __check_pxf_hdfs_read(self):
     """
     Reads the test HDFS data through PXF
@@ -155,7 +156,6 @@ class PXFServiceCheck(Script):
     headers.update(self.commonPXFHeaders)
     self.__check_pxf_read(headers)
 
-
   def __check_pxf_hdfs_write(self):
     """
     Writes some test HDFS data through PXF
@@ -184,7 +184,6 @@ class PXFServiceCheck(Script):
       msg = "PXF HDFS data write test failed"
       raise Fail(msg)
 
-
   def __cleanup_hdfs_data(self):
     """
     Cleans up the test HDFS data
@@ -201,16 +200,17 @@ class PXFServiceCheck(Script):
         )
 
 
+  # HBase Routines
   def run_hbase_tests(self):
     """
     Runs a set of PXF HBase checks
     """
     Logger.info("Running PXF HBase checks")
+    self.__cleanup_hbase_data()
     self.__check_if_client_exists("HBase")
     self.__write_hbase_data()
     self.__check_pxf_hbase_read()
 
-
   def __write_hbase_data(self):
     """
     Creates a temporary HBase table for the service checks
@@ -219,7 +219,6 @@ class PXFServiceCheck(Script):
     Execute("echo \"create '" + pxf_constants.pxf_hbase_test_table + "', 'cf'\"|hbase shell",
logoutput = True)
     Execute("echo \"put '" + pxf_constants.pxf_hbase_test_table + "', 'row1', 'cf:a', 'value1';
put '" + pxf_constants.pxf_hbase_test_table + "', 'row1', 'cf:b', 'value2'\" | hbase shell",
logoutput = True)
 
-
   def __check_pxf_hbase_read(self):
     """
     Checks reading HBase data through PXF
@@ -233,7 +232,6 @@ class PXFServiceCheck(Script):
 
     self.__check_pxf_read(headers)
 
-
   def __cleanup_hbase_data(self):
     """
     Cleans up the test HBase data
@@ -243,6 +241,49 @@ class PXFServiceCheck(Script):
     Execute("echo \"drop '" + pxf_constants.pxf_hbase_test_table + "'\"|hbase shell >
/dev/null 2>&1", logoutput = True)
 
 
+  # Hive Routines
+  def run_hive_tests(self):
+    """
+    Runs a set of PXF Hive checks
+    """
+    Logger.info("Running PXF Hive checks")
+    self.__check_if_client_exists("Hive")
+    self.__cleanup_hive_data()
+    self.__write_hive_data()
+    self.__check_pxf_hive_read()
+
+  def __write_hive_data(self):
+    """
+    Creates a temporary Hive table for the service checks
+    """
+    import params
+    Logger.info("Creating temporary Hive test data")
+    cmd = "hive -e 'CREATE TABLE IF NOT EXISTS {0} (id INT); INSERT INTO {0} VALUES (1);'".format(pxf_constants.pxf_hive_test_table)
+    Execute(cmd, logoutput = True, user = params.hive_user)
+
+  def __check_pxf_hive_read(self):
+    """
+    Checks reading Hive data through PXF
+    """
+    Logger.info("Testing PXF Hive data read")
+    headers = {
+        "X-GP-DATA-DIR": pxf_constants.pxf_hive_test_table,
+        "X-GP-profile": "Hive",
+        }
+    headers.update(self.commonPXFHeaders)
+    self.__check_pxf_read(headers)
+
+  def __cleanup_hive_data(self):
+    """
+    Cleans up the test Hive data
+    """
+    import params
+    Logger.info("Cleaning up Hive test data")
+    cmd = "hive -e 'DROP TABLE IF EXISTS {0};'".format(pxf_constants.pxf_hive_test_table)
+    Execute(cmd, logoutput = True, user = params.hive_user)
+
+
+  # Package Routines
   def __package_exists(self, pkg):
     """
     Low level function to check if a rpm is installed
@@ -250,15 +291,15 @@ class PXFServiceCheck(Script):
     if System.get_instance().os_family == "suse":
       return not runLocalCmd("zypper search " + pkg)
     else:
-      return not runLocalCmd("yum list installed | grep -i " + pkg)
+      return not runLocalCmd("yum list installed | egrep -i ^" + pkg)
 
 
   def __check_if_client_exists(self, serviceName):
     Logger.info("Checking if " + serviceName + " client libraries exist")
     if not self.__package_exists(serviceName):
-      error_msg = serviceName + "client libraries do not exist on the PXF node"
-      Logger.error(msg)
-      raise Fail(msg)
+      error_msg = serviceName + " client libraries do not exist on the PXF node"
+      Logger.error(error_msg)
+      raise Fail(error_msg)
 
 
 if __name__ == "__main__":


Mime
View raw message