ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [20/35] git commit: AMBARI-7553. Ambari needs to support WebHCat/Sqoop integration config.(vbrodetskyi)
Date Tue, 30 Sep 2014 20:48:26 GMT
AMBARI-7553. Ambari needs to support WebHCat/Sqoop integration config.(vbrodetskyi)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ba530c8d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ba530c8d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ba530c8d

Branch: refs/heads/branch-alerts-dev
Commit: ba530c8d3c7f10f40069c0b8734dbd1cbca557c6
Parents: bbed852
Author: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Authored: Tue Sep 30 09:34:08 2014 +0300
Committer: Vitaly Brodetskyi <vbrodetskyi@hortonworks.com>
Committed: Tue Sep 30 09:34:08 2014 +0300

----------------------------------------------------------------------
 .../services/HIVE/package/scripts/params.py     |  2 ++
 .../services/HIVE/package/scripts/webhcat.py    | 32 ++++++++++++++------
 .../HIVE/configuration/webhcat-site.xml         | 11 +++++++
 .../stacks/2.0.6/HIVE/test_webhcat_server.py    | 20 ++++++++++++
 4 files changed, 56 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/ba530c8d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
index e64ffa3..7e75433 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/params.py
@@ -43,6 +43,7 @@ if rpm_version is not None:
   tez_tar_file = "/usr/hdp/current/tez/lib/tez*.tar.gz"
   pig_tar_file = '/usr/hdp/current/pig/pig.tar.gz'
   hive_tar_file = '/usr/hdp/current/hive/hive.tar.gz'
+  sqoop_tar_file = '/usr/hdp/current/sqoop/sqoop*.tar.gz'
 
   hcat_lib = '/usr/hdp/current/hive/hive-hcatalog/share/hcatalog'
   webhcat_bin_dir = '/usr/hdp/current/hive-hcatalog/sbin'
@@ -58,6 +59,7 @@ else:
   tez_tar_file = "/usr/lib/tez/tez*.tar.gz"
   pig_tar_file = '/usr/share/HDP-webhcat/pig.tar.gz'
   hive_tar_file = '/usr/share/HDP-webhcat/hive.tar.gz'
+  sqoop_tar_file = '/usr/share/HDP-webhcat/sqoop*.tar.gz'
 
   if str(hdp_stack_version).startswith('2.0'):
     hcat_lib = '/usr/lib/hcatalog/share/hcatalog'

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba530c8d/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
index 6ce47f5..b034cbc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/webhcat.py
@@ -20,6 +20,8 @@ Ambari Agent
 """
 from resource_management import *
 import sys
+import os.path
+import glob
 
 
 def webhcat():
@@ -95,15 +97,16 @@ def webhcat():
                 hadoop_conf_dir=params.hadoop_conf_dir
   )
 
-  CopyFromLocal(params.pig_tar_file,
-                owner=params.webhcat_user,
-                mode=0755,
-                dest_dir=params.webhcat_apps_dir,
-                kinnit_if_needed=kinit_if_needed,
-                hdfs_user=params.hdfs_user,
-                hadoop_bin_dir=params.hadoop_bin_dir,
-                hadoop_conf_dir=params.hadoop_conf_dir
-  )
+  if (os.path.isfile(params.pig_tar_file)):
+    CopyFromLocal(params.pig_tar_file,
+                  owner=params.webhcat_user,
+                  mode=0755,
+                  dest_dir=params.webhcat_apps_dir,
+                  kinnit_if_needed=kinit_if_needed,
+                  hdfs_user=params.hdfs_user,
+                  hadoop_bin_dir=params.hadoop_bin_dir,
+                  hadoop_conf_dir=params.hadoop_conf_dir
+    )
 
   CopyFromLocal(params.hive_tar_file,
                 owner=params.webhcat_user,
@@ -114,3 +117,14 @@ def webhcat():
                 hadoop_bin_dir=params.hadoop_bin_dir,
                 hadoop_conf_dir=params.hadoop_conf_dir
   )
+
+  if (len(glob.glob(params.sqoop_tar_file)) > 0):
+    CopyFromLocal(params.sqoop_tar_file,
+                  owner=params.webhcat_user,
+                  mode=0755,
+                  dest_dir=params.webhcat_apps_dir,
+                  kinnit_if_needed=kinit_if_needed,
+                  hdfs_user=params.hdfs_user,
+                  hadoop_bin_dir=params.hadoop_bin_dir,
+                  hadoop_conf_dir=params.hadoop_conf_dir
+    )

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba530c8d/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
index bdecb67..f0d6a84 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/HIVE/configuration/webhcat-site.xml
@@ -49,5 +49,16 @@ limitations under the License.
     <description>The path to the hcatalog executable.</description>
   </property>
 
+  <property>
+    <name>templeton.sqoop.archive</name>
+    <value>hdfs:///apps/webhcat/sqoop.tar.gz</value>
+    <description>The path to the Sqoop archive in HDFS.</description>
+  </property>
+
+  <property>
+    <name>templeton.sqoop.path</name>
+    <value>sqoop.tar.gz/sqoop/bin/sqoop</value>
+    <description>The path to the Sqoop executable.</description>
+  </property>
 
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/ba530c8d/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
index 31588b8..5f92a2d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HIVE/test_webhcat_server.py
@@ -20,6 +20,8 @@ limitations under the License.
 from mock.mock import MagicMock, patch
 from stacks.utils.RMFTestCase import *
 
+@patch("os.path.isfile", new = MagicMock(return_value=True))
+@patch("glob.glob", new = MagicMock(return_value=["one", "two"]))
 class TestWebHCatServer(RMFTestCase):
 
   def test_configure_default(self):
@@ -183,6 +185,15 @@ class TestWebHCatServer(RMFTestCase):
                               hadoop_conf_dir='/etc/hadoop/conf',
                               hdfs_user='hdfs'
     )
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/sqoop*.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir='/apps/webhcat',
+                              kinnit_if_needed='',
+                              hadoop_bin_dir='/usr/bin',
+                              hadoop_conf_dir='/etc/hadoop/conf',
+                              hdfs_user='hdfs'
+    )
 
   def assert_configure_secured(self):
     self.assertResourceCalled('HdfsDirectory', '/apps/webhcat',
@@ -275,4 +286,13 @@ class TestWebHCatServer(RMFTestCase):
                               hadoop_conf_dir='/etc/hadoop/conf',
                               hadoop_bin_dir='/usr/bin',
                               hdfs_user='hdfs'
+    )
+    self.assertResourceCalled('CopyFromLocal', '/usr/share/HDP-webhcat/sqoop*.tar.gz',
+                              owner='hcat',
+                              mode=0755,
+                              dest_dir='/apps/webhcat',
+                              kinnit_if_needed='/usr/bin/kinit -kt /etc/security/keytabs/hdfs.headless.keytab
hdfs;',
+                              hadoop_conf_dir='/etc/hadoop/conf',
+                              hadoop_bin_dir='/usr/bin',
+                              hdfs_user='hdfs'
     )
\ No newline at end of file


Mime
View raw message