ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [1/2] git commit: AMBARI-5254. Installed HDP 2.0 Stack but Tez user created (aonishuk)
Date Thu, 27 Mar 2014 21:27:23 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-1.5.0 ec7f572d4 -> 4b28e139f
  refs/heads/trunk c06183ba7 -> 5ef9adce2


AMBARI-5254. Installed HDP 2.0 Stack but Tez user created (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5ef9adce
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5ef9adce
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5ef9adce

Branch: refs/heads/trunk
Commit: 5ef9adce2186ea676979c5b32f334d13a1e7ed03
Parents: c06183b
Author: Andrew Onischuk <aonishuk@hortonworks.com>
Authored: Thu Mar 27 14:24:19 2014 -0700
Committer: Andrew Onischuk <aonishuk@hortonworks.com>
Committed: Thu Mar 27 14:24:19 2014 -0700

----------------------------------------------------------------------
 .../hooks/before-INSTALL/scripts/params.py      |   1 +
 .../scripts/shared_initialization.py            |  10 +-
 .../services/HIVE/configuration/global.xml      |   5 -
 .../HIVE/package/scripts/hive_server.py         | 133 +------------------
 .../HIVE/package/scripts/install_jars.py        | 131 ++++++++++++++++++
 5 files changed, 144 insertions(+), 136 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/5ef9adce/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
index 7ca5512..341d86f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/params.py
@@ -75,6 +75,7 @@ has_zk_host = not len(zk_hosts) == 0
 has_ganglia_server = not len(ganglia_server_hosts) == 0
 has_storm_server = not len(storm_server_hosts) == 0
 has_falcon_server = not len(falcon_host) == 0
+has_tez = 'tez-site' in config['configurations']
 
 is_namenode_master = hostname in namenode_host
 is_jtnode_master = hostname in jtnode_host

http://git-wip-us.apache.org/repos/asf/ambari/blob/5ef9adce/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
index f786eed..c76c028 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
@@ -34,10 +34,6 @@ def setup_users():
        gid=params.user_group,
        groups=[params.proxyuser_group]
   )
-  User(params.tez_user,
-      gid=params.user_group,
-      groups=[params.proxyuser_group]
-  )
   smoke_user_dirs = format(
     "/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}")
   set_uid(params.smoke_user, smoke_user_dirs)
@@ -106,6 +102,12 @@ def setup_users():
          gid=params.user_group,
          groups=[params.user_group]
     )
+    
+  if params.has_tez:  
+    User(params.tez_user,
+      gid=params.user_group,
+      groups=[params.proxyuser_group]
+  )
 
 def set_uid(user, user_dirs):
   """

http://git-wip-us.apache.org/repos/asf/ambari/blob/5ef9adce/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/global.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/global.xml
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/global.xml
index 6841f59..ae7f586 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/global.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/configuration/global.xml
@@ -121,11 +121,6 @@
     <value>hive</value>
     <description>Hive User.</description>
   </property>
-  <property>
-    <name>tez_user</name>
-    <value>tez</value>
-    <description></description>
-  </property>
 
   <!--HCAT-->
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/5ef9adce/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
index a1a9256..2105a90 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive_server.py
@@ -21,8 +21,8 @@ limitations under the License.
 from resource_management import *
 from hive import hive
 from hive_service import hive_service
-import os
-import fnmatch
+from install_jars import install_hive_exec_jar
+from install_jars import install_tez_jars
 
 class HiveServer(Script):
 
@@ -39,8 +39,10 @@ class HiveServer(Script):
     import params
     env.set_params(params)
     self.configure(env) # FOR SECURITY
-    self.install_tez_jars(params) # Put tez jars in hdfs
-    self.install_hive_exec_jar(params) # Put hive exec jar in hdfs
+    
+    install_tez_jars() # Put tez jars in hdfs
+    install_hive_exec_jar() # Put hive exec jar in hdfs
+    
     hive_service( 'hiveserver2',
                   action = 'start'
     )
@@ -60,129 +62,6 @@ class HiveServer(Script):
     # Recursively check all existing gmetad pid files
     check_process_status(pid_file)
 
-  def install_hive_exec_jar(self, params):
-    hdfs_path_prefix = 'hdfs://'
-    if params.tez_lib_uris:
-      hdfs_path = params.hive_exec_hdfs_path
-
-      if hdfs_path.strip().find(hdfs_path_prefix, 0) != -1:
-        hdfs_path = hdfs_path.replace(hdfs_path_prefix, '')
-      pass
-
-      params.HdfsDirectory(hdfs_path,
-                           action="create",
-                           owner=params.hive_user,
-                           mode=0755
-      )
-
-      if params.security_enabled:
-        kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_user};")
-      else:
-        kinit_if_needed = ""
-
-      if kinit_if_needed:
-        Execute(kinit_if_needed,
-                user=params.tez_user,
-                path='/bin'
-        )
-
-      hive_exec_jar_path = self.find_hive_exec_jar_path(params.hive_lib)
-      if hive_exec_jar_path is None:
-        hive_exec_jar_path = params.hive_exec_jar_path
-      pass
-
-      CopyFromLocal(hive_exec_jar_path,
-                    mode=0755,
-                    owner=params.hive_user,
-                    dest_dir=hdfs_path,
-                    kinnit_if_needed=kinit_if_needed,
-                    hdfs_user=params.hdfs_user
-      )
-    pass
-
-  def find_hive_exec_jar_path(self, hive_lib_dir):
-    if os.path.exists(hive_lib_dir) and os.path.isdir(hive_lib_dir):
-      for file in os.listdir(hive_lib_dir):
-        file_path = os.path.join(hive_lib_dir, file)
-        if fnmatch.fnmatch(file, 'hive-exec*.jar') and not os.path.islink(file_path):
-          return file_path
-      pass
-    pass
-
-  def install_tez_jars(self, params):
-    destination_hdfs_dirs = get_tez_hdfs_dir_paths(params.tez_lib_uris)
-
-    # If tez libraries are to be stored in hdfs
-    if destination_hdfs_dirs:
-      for hdfs_dir in destination_hdfs_dirs:
-        params.HdfsDirectory(hdfs_dir,
-                            action="create_delayed",
-                            owner=params.tez_user,
-                            mode=0755
-        )
-      pass
-      params.HdfsDirectory(None, action="create")
-
-      if params.security_enabled:
-        kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_user};")
-      else:
-        kinit_if_needed = ""
-
-      if kinit_if_needed:
-        Execute(kinit_if_needed,
-                user=params.tez_user,
-                path='/bin'
-        )
-      pass
-
-      app_dir_path = None
-      lib_dir_path = None
-
-      if len(destination_hdfs_dirs) > 1:
-        for path in destination_hdfs_dirs:
-          if 'lib' in path:
-            lib_dir_path = path
-          else:
-            app_dir_path = path
-          pass
-        pass
-      pass
-
-      if app_dir_path:
-        CopyFromLocal(params.tez_local_api_jars,
-                      mode=0755,
-                      owner=params.tez_user,
-                      dest_dir=app_dir_path,
-                      kinnit_if_needed=kinit_if_needed,
-                      hdfs_user=params.hdfs_user
-        )
-      pass
-
-      if lib_dir_path:
-        CopyFromLocal(params.tez_local_lib_jars,
-                      mode=0755,
-                      owner=params.tez_user,
-                      dest_dir=lib_dir_path,
-                      kinnit_if_needed=kinit_if_needed,
-                      hdfs_user=params.hdfs_user
-        )
-      pass
-
-
-def get_tez_hdfs_dir_paths(tez_lib_uris = None):
-  hdfs_path_prefix = 'hdfs://'
-  lib_dir_paths = []
-  if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1:
-    dir_paths = tez_lib_uris.split(',')
-    for path in dir_paths:
-      lib_dir_path = path.replace(hdfs_path_prefix, '')
-      lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
-      lib_dir_paths.append(lib_dir_path)
-    pass
-  pass
-
-  return lib_dir_paths
-
 
 if __name__ == "__main__":
   HiveServer().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/5ef9adce/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
new file mode 100644
index 0000000..9495b30
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/install_jars.py
@@ -0,0 +1,131 @@
+from resource_management import *
+
+import os
+import fnmatch
+
+def install_hive_exec_jar():
+  import params
+  
+  hdfs_path_prefix = 'hdfs://'
+  if params.tez_lib_uris:
+    hdfs_path = params.hive_exec_hdfs_path
+
+    if hdfs_path.strip().find(hdfs_path_prefix, 0) != -1:
+      hdfs_path = hdfs_path.replace(hdfs_path_prefix, '')
+    pass
+
+    params.HdfsDirectory(hdfs_path,
+                         action="create",
+                         owner=params.hive_user,
+                         mode=0755
+    )
+
+    if params.security_enabled:
+      kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_user};")
+    else:
+      kinit_if_needed = ""
+
+    if kinit_if_needed:
+      Execute(kinit_if_needed,
+              user=params.tez_user,
+              path='/bin'
+      )
+
+    hive_exec_jar_path = find_hive_exec_jar_path(params.hive_lib)
+    if hive_exec_jar_path is None:
+      hive_exec_jar_path = params.hive_exec_jar_path
+    pass
+
+    CopyFromLocal(hive_exec_jar_path,
+                  mode=0755,
+                  owner=params.hive_user,
+                  dest_dir=hdfs_path,
+                  kinnit_if_needed=kinit_if_needed,
+                  hdfs_user=params.hdfs_user
+    )
+  pass
+
+def find_hive_exec_jar_path(hive_lib_dir):
+  if os.path.exists(hive_lib_dir) and os.path.isdir(hive_lib_dir):
+    for file in os.listdir(hive_lib_dir):
+      file_path = os.path.join(hive_lib_dir, file)
+      if fnmatch.fnmatch(file, 'hive-exec*.jar') and not os.path.islink(file_path):
+        return file_path
+    pass
+  pass
+
+def install_tez_jars():
+  import params
+  
+  destination_hdfs_dirs = get_tez_hdfs_dir_paths(params.tez_lib_uris)
+
+  # If tez libraries are to be stored in hdfs
+  if destination_hdfs_dirs:
+    for hdfs_dir in destination_hdfs_dirs:
+      params.HdfsDirectory(hdfs_dir,
+                          action="create_delayed",
+                          owner=params.tez_user,
+                          mode=0755
+      )
+    pass
+    params.HdfsDirectory(None, action="create")
+
+    if params.security_enabled:
+      kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_user};")
+    else:
+      kinit_if_needed = ""
+
+    if kinit_if_needed:
+      Execute(kinit_if_needed,
+              user=params.tez_user,
+              path='/bin'
+      )
+    pass
+
+    app_dir_path = None
+    lib_dir_path = None
+
+    if len(destination_hdfs_dirs) > 1:
+      for path in destination_hdfs_dirs:
+        if 'lib' in path:
+          lib_dir_path = path
+        else:
+          app_dir_path = path
+        pass
+      pass
+    pass
+
+    if app_dir_path:
+      CopyFromLocal(params.tez_local_api_jars,
+                    mode=0755,
+                    owner=params.tez_user,
+                    dest_dir=app_dir_path,
+                    kinnit_if_needed=kinit_if_needed,
+                    hdfs_user=params.hdfs_user
+      )
+    pass
+
+    if lib_dir_path:
+      CopyFromLocal(params.tez_local_lib_jars,
+                    mode=0755,
+                    owner=params.tez_user,
+                    dest_dir=lib_dir_path,
+                    kinnit_if_needed=kinit_if_needed,
+                    hdfs_user=params.hdfs_user
+      )
+    pass
+
+
+def get_tez_hdfs_dir_paths(tez_lib_uris = None):
+  hdfs_path_prefix = 'hdfs://'
+  lib_dir_paths = []
+  if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1:
+    dir_paths = tez_lib_uris.split(',')
+    for path in dir_paths:
+      lib_dir_path = path.replace(hdfs_path_prefix, '')
+      lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
+      lib_dir_paths.append(lib_dir_path)
+    pass
+  pass
+
+  return lib_dir_paths
\ No newline at end of file


Mime
View raw message