ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From vsai...@apache.org
Subject ambari git commit: AMBARI-21835.Support for Zeppelin notebook storage in HDFS(Prabhjyot Singh via Venkata Sairam)
Date Mon, 04 Sep 2017 09:15:31 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 7e222173b -> a679281f7


AMBARI-21835.Support for Zeppelin notebook storage in HDFS(Prabhjyot Singh via Venkata Sairam)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a679281f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a679281f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a679281f

Branch: refs/heads/trunk
Commit: a679281f7abe6768e0896e32df7c298b59877064
Parents: 7e22217
Author: Venkata Sairam <venkatasairam.lanka@gmail.com>
Authored: Mon Sep 4 14:44:25 2017 +0530
Committer: Venkata Sairam <venkatasairam.lanka@gmail.com>
Committed: Mon Sep 4 14:45:22 2017 +0530

----------------------------------------------------------------------
 .../0.6.0.2.5/configuration/zeppelin-config.xml |  2 +-
 .../0.6.0.2.5/package/scripts/master.py         | 59 +++++++++++++++++---
 .../0.6.0.2.5/package/scripts/params.py         |  2 +
 .../0.6.0.3.0/configuration/zeppelin-config.xml |  2 +-
 4 files changed, 55 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
index 03ad5f7..bd6ad76 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-config.xml
@@ -77,7 +77,7 @@
   </property>
   <property>
     <name>zeppelin.notebook.storage</name>
-    <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
+    <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
     <description>notebook persistence layer implementation. If S3 is used, set this
to
             org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store
the
             notebooks, it is necessary to use the following folder structure

http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 2142bb4..ba73d10 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -20,25 +20,25 @@ limitations under the License.
 
 import glob
 import os
-from resource_management.core.base import Fail
+
+from resource_management.core import shell, sudo
+from resource_management.core.logger import Logger
 from resource_management.core.resources import Directory
 from resource_management.core.resources.system import Execute, File
 from resource_management.core.source import InlineTemplate
-from resource_management.core import sudo
-from resource_management.core.logger import Logger
-from resource_management.core.source import StaticFile
 from resource_management.libraries import XmlConfig
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.format import format
-from resource_management.libraries.functions import stack_select
-from resource_management.libraries.functions import StackFeature
-from resource_management.libraries.functions.decorator import retry
 from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.script.script import Script
 
-class Master(Script):
 
+class Master(Script):
   def install(self, env):
     import params
     env.set_params(params)
@@ -185,6 +185,45 @@ class Master(Script):
                 group=params.zeppelin_group,
                 mode=0644)
 
+  def check_and_copy_notebook_in_hdfs(self, params):
+    if params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir'].startswith("/"):
+      notebook_directory = params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir']
+    else:
+      notebook_directory = "/user/" + format("{zeppelin_user}") + "/" + \
+                           params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir']
+
+    kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
+    kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
+
+    notebook_directory_exists = shell.call(format("{kinit_if_needed} hdfs --config {hadoop_conf_dir}
dfs -test -e {notebook_directory};echo $?"),
+                                           user=params.zeppelin_user)[1]
+
+    #if there is no kerberos setup then the string will contain "-bash: kinit: command not
found"
+    if "\n" in notebook_directory_exists:
+      notebook_directory_exists = notebook_directory_exists.split("\n")[1]
+
+    # '1' means it does not exists
+    if notebook_directory_exists == '1':
+      # hdfs dfs -mkdir {notebook_directory}
+      params.HdfsResource(format("{notebook_directory}"),
+                          type="directory",
+                          action="create_on_execute",
+                          owner=params.zeppelin_user,
+                          recursive_chown=True,
+                          recursive_chmod=True
+                          )
+
+      # hdfs dfs -put /usr/hdp/current/zeppelin-server/notebook/ {notebook_directory}
+      params.HdfsResource(format("{notebook_directory}"),
+                            type="directory",
+                            action="create_on_execute",
+                            source=params.notebook_dir,
+                            owner=params.zeppelin_user,
+                            recursive_chown=True,
+                            recursive_chmod=True
+                            )
+
+
   def stop(self, env, upgrade_type=None):
     import params
     self.create_zeppelin_log_dir(env)
@@ -202,6 +241,10 @@ class Master(Script):
     Execute(("chown", "-R", format("{zeppelin_user}") + ":" + format("{zeppelin_group}"),
              os.path.join(params.zeppelin_dir, "notebook")), sudo=True)
 
+    if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config']
\
+        and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage']
== 'org.apache.zeppelin.notebook.repo.HdfsNotebookRepo':
+      self.check_and_copy_notebook_in_hdfs(params)
+
     if params.security_enabled:
         zeppelin_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};
")
         Execute(zeppelin_kinit_cmd, user=params.zeppelin_user)

http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
index f5a2a37..3242f26 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
@@ -62,6 +62,8 @@ executor_mem = config['configurations']['zeppelin-env']['zeppelin.executor.mem']
 executor_instances = config['configurations']['zeppelin-env'][
   'zeppelin.executor.instances']
 
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
 spark_jar_dir = config['configurations']['zeppelin-env']['zeppelin.spark.jar.dir']
 spark_jar = format("{spark_jar_dir}/zeppelin-spark-0.5.5-SNAPSHOT.jar")
 setup_view = True

http://git-wip-us.apache.org/repos/asf/ambari/blob/a679281f/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
index 2bfc0dc..ca6b295 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.3.0/configuration/zeppelin-config.xml
@@ -77,7 +77,7 @@
   </property>
   <property>
     <name>zeppelin.notebook.storage</name>
-    <value>org.apache.zeppelin.notebook.repo.VFSNotebookRepo</value>
+    <value>org.apache.zeppelin.notebook.repo.HdfsNotebookRepo</value>
     <description>notebook persistence layer implementation. If S3 is used, set this
to
             org.apache.zeppelin.notebook.repo.S3NotebookRepo instead. If S3 is used to store
the
             notebooks, it is necessary to use the following folder structure


Mime
View raw message