ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dmitriu...@apache.org
Subject [2/2] git commit: AMBARI-7863. Ambari should support for Pig-on-Tez (dlysnichenko)
Date Tue, 21 Oct 2014 10:21:06 GMT
AMBARI-7863. Ambari should support for Pig-on-Tez (dlysnichenko)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/64016a50
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/64016a50
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/64016a50

Branch: refs/heads/branch-1.7.0
Commit: 64016a50e5d9e82ee2366e9619945950aa818d17
Parents: d4e8b56
Author: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Authored: Tue Oct 21 13:19:50 2014 +0300
Committer: Lisnichenko Dmitro <dlysnichenko@hortonworks.com>
Committed: Tue Oct 21 13:20:49 2014 +0300

----------------------------------------------------------------------
 .../services/HDFS/package/scripts/params.py     |  2 +-
 .../services/HDFS/package/scripts/utils.py      |  6 +--
 .../services/PIG/package/scripts/params.py      |  3 ++
 .../PIG/package/scripts/service_check.py        | 29 ++++++++++++++
 .../stacks/HDP/2.2/services/PIG/metainfo.xml    |  3 ++
 .../python/stacks/2.0.6/PIG/test_pig_client.py  | 40 ++++++++++++++++++++
 6 files changed, 79 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
index 1308aa2..6b16205 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/params.py
@@ -212,7 +212,7 @@ hadoop_env_sh_template = config['configurations']['hadoop-env']['content']
 java_home = config['hostLevelParams']['java_home']
 stack_version = str(config['hostLevelParams']['stack_version'])
 
-stack_is_champlain_or_further = not (stack_version.startswith('2.0') or stack_version.startswith('2.1'))
+stack_is_hdp22_or_further = not (stack_version.startswith('2.0') or stack_version.startswith('2.1'))
 
 if stack_version.startswith('2.0') and System.get_instance().os_family != "suse":
   # deprecated rhel jsvc_path

http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
index 5e1221a..14251cd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/utils.py
@@ -75,14 +75,14 @@ def service(action=None, name=None, user=None, create_pid_dir=False,
     hadoop_secure_dn_pid_file = format("{hadoop_secure_dn_pid_dir}/hadoop_secure_dn.pid")
 
     # At Champlain stack and further, we may start datanode as a non-root even in secure
cluster
-    if not params.stack_is_champlain_or_further or secure_ports_are_in_use:
+    if not params.stack_is_hdp22_or_further or secure_ports_are_in_use:
       user = "root"
       pid_file = format(
         "{hadoop_pid_dir_prefix}/{hdfs_user}/hadoop-{hdfs_user}-{name}.pid")
-      if params.stack_is_champlain_or_further:
+      if params.stack_is_hdp22_or_further:
         hadoop_env_exports.update(hadoop_secure_dn_exports)
 
-    if action == 'stop' and params.stack_is_champlain_or_further and \
+    if action == 'stop' and params.stack_is_hdp22_or_further and \
       os.path.isfile(hadoop_secure_dn_pid_file):
         # We need special handling for this case to handle the situation
         # when we configure non-root secure DN and then restart it

http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
index 785b89d..f28e541 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
@@ -55,3 +55,6 @@ java64_home = config['hostLevelParams']['java_home']
 pig_properties = config['configurations']['pig-properties']['content']
 
 log4j_props = config['configurations']['pig-log4j']['content']
+
+stack_version = str(config['hostLevelParams']['stack_version'])
+stack_is_hdp22_or_further = not (stack_version.startswith('2.0') or stack_version.startswith('2.1'))
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
index 7619bd6..88c46e2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
@@ -51,6 +51,7 @@ class PigServiceCheck(Script):
       mode = 0755
     )
 
+    # check for Pig-on-M/R
     Execute( format("pig {tmp_dir}/pigSmoke.sh"),
       tries     = 3,
       try_sleep = 5,
@@ -64,6 +65,34 @@ class PigServiceCheck(Script):
       bin_dir = params.hadoop_bin_dir
     )
 
+    if params.stack_is_hdp22_or_further:
+      # cleanup results from previous test
+      ExecuteHadoop( create_file_cmd,
+        tries     = 3,
+        try_sleep = 5,
+        user      = params.smokeuser,
+        conf_dir = params.hadoop_conf_dir,
+        # for kinit run
+        keytab = params.smoke_user_keytab,
+        security_enabled = params.security_enabled,
+        kinit_path_local = params.kinit_path_local,
+        bin_dir = params.hadoop_bin_dir
+      )
+
+      # Check for Pig-on-Tez
+      Execute( format("pig -x tez {tmp_dir}/pigSmoke.sh"),
+        tries     = 3,
+        try_sleep = 5,
+        path      = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+        user      = params.smokeuser
+      )
+
+      ExecuteHadoop( test_cmd,
+        user      = params.smokeuser,
+        conf_dir = params.hadoop_conf_dir,
+        bin_dir = params.hadoop_bin_dir
+      )
+
 if __name__ == "__main__":
   PigServiceCheck().execute()
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml
index f43d395..dff2ae7 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/PIG/metainfo.xml
@@ -40,6 +40,9 @@
           </packages>
         </osSpecific>
       </osSpecifics>
+      <requiredServices>
+        <service>TEZ</service>
+      </requiredServices>
     </service>
   </services>
 </metainfo>

http://git-wip-us.apache.org/repos/asf/ambari/blob/64016a50/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
index 2874614..51eabbd 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
@@ -19,6 +19,7 @@ limitations under the License.
 '''
 
 from stacks.utils.RMFTestCase import *
+import json
 
 class TestPigClient(RMFTestCase):
 
@@ -52,6 +53,8 @@ class TestPigClient(RMFTestCase):
     )
     self.assertNoMoreResources()
 
+
+
   def test_configure_secured(self):
     self.executeScript("2.0.6/services/PIG/package/scripts/pig_client.py",
                        classname = "PigClient",
@@ -81,3 +84,40 @@ class TestPigClient(RMFTestCase):
       content = 'log4jproperties\nline2'
     )
     self.assertNoMoreResources()
+
+  def test_configure_default_hdp22(self):
+
+    config_file = "stacks/2.0.6/configs/default.json"
+    with open(config_file, "r") as f:
+      default_json = json.load(f)
+
+    default_json['hostLevelParams']['stack_version'] = '2.2'
+
+    self.executeScript("2.0.6/services/PIG/package/scripts/pig_client.py",
+                       classname = "PigClient",
+                       command = "configure",
+                       config_dict=default_json
+    )
+
+    self.assertResourceCalled('Directory', '/etc/pig/conf',
+                              recursive = True,
+                              owner = 'hdfs',
+                              group = 'hadoop'
+    )
+    self.assertResourceCalled('File', '/etc/pig/conf/pig-env.sh',
+                              owner = 'hdfs',
+                              content = InlineTemplate(self.getConfig()['configurations']['pig-env']['content'])
+    )
+    self.assertResourceCalled('File', '/etc/pig/conf/pig.properties',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              mode = 0644,
+                              content = 'pigproperties\nline2'
+    )
+    self.assertResourceCalled('File', '/etc/pig/conf/log4j.properties',
+                              owner = 'hdfs',
+                              group = 'hadoop',
+                              mode = 0644,
+                              content = 'log4jproperties\nline2'
+    )
+    self.assertNoMoreResources()
\ No newline at end of file


Mime
View raw message