ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From lpus...@apache.org
Subject [03/50] [abbrv] ambari git commit: AMBARI-22199.Zeppelin start fails due to permission denied error during kinit(Venkata Sairam)
Date Thu, 12 Oct 2017 17:27:04 GMT
AMBARI-22199.Zeppelin start fails due to permission denied error during kinit(Venkata Sairam)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/33d67f3e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/33d67f3e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/33d67f3e

Branch: refs/heads/feature-branch-AMBARI-21307
Commit: 33d67f3ead24fcda4fcdb7d5920ccc76d1743657
Parents: 7e885a3
Author: Venkata Sairam <venkatasairam.lanka@gmail.com>
Authored: Wed Oct 11 14:46:39 2017 +0530
Committer: Venkata Sairam <venkatasairam.lanka@gmail.com>
Committed: Wed Oct 11 14:46:39 2017 +0530

----------------------------------------------------------------------
 .../ZEPPELIN/0.7.0/package/scripts/master.py    |  48 ++++---
 .../stacks/2.6/ZEPPELIN/test_zeppelin_070.py    | 140 ++++++++++++++++---
 2 files changed, 147 insertions(+), 41 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/33d67f3e/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
index d615d06..09944bd 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
@@ -192,8 +192,7 @@ class Master(Script):
       notebook_directory = "/user/" + format("{zeppelin_user}") + "/" + \
                            params.config['configurations']['zeppelin-config']['zeppelin.notebook.dir']
 
-
-    if self.is_path_exists_in_HDFS(notebook_directory, params.zeppelin_user):
+    if not self.is_path_exists_in_HDFS(notebook_directory, params.zeppelin_user):
       # hdfs dfs -mkdir {notebook_directory}
       params.HdfsResource(format("{notebook_directory}"),
                           type="directory",
@@ -231,14 +230,14 @@ class Master(Script):
     Execute(("chown", "-R", format("{zeppelin_user}") + ":" + format("{zeppelin_group}"),
              os.path.join(params.zeppelin_dir, "notebook")), sudo=True)
 
+    if params.security_enabled:
+      zeppelin_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};
")
+      Execute(zeppelin_kinit_cmd, user=params.zeppelin_user)
+
     if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config']
\
         and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage']
== 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
       self.check_and_copy_notebook_in_hdfs(params)
 
-    if params.security_enabled:
-        zeppelin_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};
")
-        Execute(zeppelin_kinit_cmd, user=params.zeppelin_user)
-
     zeppelin_spark_dependencies = self.get_zeppelin_spark_dependencies()
     if zeppelin_spark_dependencies and os.path.exists(zeppelin_spark_dependencies[0]):
       self.create_zeppelin_dir(params)
@@ -292,14 +291,17 @@ class Master(Script):
     if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, format_stack_version(params.version)):
       stack_select.select_packages(params.version)
 
-  def getZeppelinConfFS(self, params):
-    hdfs_interpreter_config = params.config['configurations']['zeppelin-config']['zeppelin.config.fs.dir']
+ "/interpreter.json"
+  def get_zeppelin_conf_FS_directory(self, params):
+    hdfs_interpreter_config = params.config['configurations']['zeppelin-config']['zeppelin.config.fs.dir']
 
     if not hdfs_interpreter_config.startswith("/"):
       hdfs_interpreter_config = "/user/" + format("{zeppelin_user}") + "/" + hdfs_interpreter_config
 
     return hdfs_interpreter_config
 
+  def get_zeppelin_conf_FS(self, params):
+    return self.get_zeppelin_conf_FS_directory(params) + "/interpreter.json"
+
   def is_path_exists_in_HDFS(self, path, as_user):
     kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',
None))
     kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
@@ -325,17 +327,15 @@ class Master(Script):
       and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage']
== 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
 
       if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
-        zeppelin_conf_fs = self.getZeppelinConfFS(params)
+        zeppelin_conf_fs = self.get_zeppelin_conf_FS(params)
 
         if self.is_path_exists_in_HDFS(zeppelin_conf_fs, params.zeppelin_user):
           # copy from hdfs to /etc/zeppelin/conf/interpreter.json
-          params.HdfsResource(interpreter_config,
-                              type="file",
-                              action="download_on_execute",
-                              source=zeppelin_conf_fs,
-                              user=params.zeppelin_user,
-                              group=params.zeppelin_group,
-                              owner=params.zeppelin_user)
+          kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',None))
+          kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
+          shell.call(format("rm {interpreter_config};"
+                            "{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -get {zeppelin_conf_fs}
{interpreter_config}"),
+                     user=params.zeppelin_user)
         else:
           Logger.info(format("{zeppelin_conf_fs} does not exist. Skipping upload of DFS."))
 
@@ -357,13 +357,23 @@ class Master(Script):
       and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage']
== 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
 
       if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
-        params.HdfsResource(self.getZeppelinConfFS(params),
+        if not self.is_path_exists_in_HDFS(self.get_zeppelin_conf_FS_directory(params), params.zeppelin_user):
+          # hdfs dfs -mkdir {zeppelin's conf directory}
+          params.HdfsResource(self.get_zeppelin_conf_FS_directory(params),
+                              type="directory",
+                              action="create_on_execute",
+                              owner=params.zeppelin_user,
+                              recursive_chown=True,
+                              recursive_chmod=True
+                          )
+
+        params.HdfsResource(self.get_zeppelin_conf_FS(params),
                             type="file",
                             action="create_on_execute",
                             source=interpreter_config,
-                            user=params.zeppelin_user,
-                            group=params.zeppelin_group,
                             owner=params.zeppelin_user,
+                            recursive_chown=True,
+                            recursive_chmod=True,
                             replace_existing_files=True)
 
   def update_kerberos_properties(self):

http://git-wip-us.apache.org/repos/asf/ambari/blob/33d67f3e/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
index 3064880..3adb94c 100644
--- a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
+++ b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
@@ -221,6 +221,42 @@ class TestZeppelin070(RMFTestCase):
                                           '/usr/hdp/current/zeppelin-server/notebook'),
                               sudo=True,
                               )
+    self.assertResourceCalled('HdfsResource', '/user/zeppelin/notebook',
+                              hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+                              default_fs=u'hdfs://c6401.ambari.apache.org:8020',
+                              hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              hdfs_site={u'a': u'b'},
+                              keytab=UnknownConfigurationMock(),
+                              kinit_path_local='/usr/bin/kinit',
+                              user='hdfs',
+                              owner='zeppelin',
+                              principal_name=UnknownConfigurationMock(),
+                              recursive_chown=True,
+                              security_enabled=False,
+                              hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+                              type='directory',
+                              action=['create_on_execute'],
+                              recursive_chmod=True
+                              )
+    self.assertResourceCalled('HdfsResource', '/user/zeppelin/notebook',
+                              hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+                              default_fs=u'hdfs://c6401.ambari.apache.org:8020',
+                              hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              hdfs_site={u'a': u'b'},
+                              keytab=UnknownConfigurationMock(),
+                              kinit_path_local='/usr/bin/kinit',
+                              user='hdfs',
+                              owner='zeppelin',
+                              principal_name=UnknownConfigurationMock(),
+                              recursive_chown=True,
+                              security_enabled=False,
+                              source='/usr/hdp/current/zeppelin-server/notebook',
+                              hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+                              type='directory',
+                              action=['create_on_execute'],
+                              recursive_chmod=True
+                              )
+
     self.assertResourceCalled('HdfsResource', '/user/zeppelin',
                               hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
                               default_fs=u'hdfs://c6401.ambari.apache.org:8020',
@@ -312,6 +348,25 @@ class TestZeppelin070(RMFTestCase):
                           )
 
     self.assertResourceCalled('HdfsResource',
+                              '/user/zeppelin/hdfs:///user/zeppelin/conf',
+                              security_enabled=False,
+                              hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+                              keytab=UnknownConfigurationMock(),
+                              default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              hdfs_site={u'a': u'b'},
+                              kinit_path_local='/usr/bin/kinit',
+                              principal_name=UnknownConfigurationMock(),
+                              recursive_chown=True,
+                              recursive_chmod=True,
+                              owner='zeppelin',
+                              user='hdfs',
+                              hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+                              type='directory',
+                              action=['create_on_execute'],
+                              )
+
+    self.assertResourceCalled('HdfsResource',
                               '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
                               security_enabled=False,
                               hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
@@ -322,9 +377,10 @@ class TestZeppelin070(RMFTestCase):
                               hdfs_site={u'a': u'b'},
                               kinit_path_local='/usr/bin/kinit',
                               principal_name=UnknownConfigurationMock(),
-                              user='zeppelin',
+                              recursive_chown=True,
+                              recursive_chmod=True,
+                              user='hdfs',
                               owner='zeppelin',
-                              group='zeppelin',
                               replace_existing_files=True,
                               hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
                               type='file',
@@ -337,24 +393,45 @@ class TestZeppelin070(RMFTestCase):
                               group='zeppelin',
                               )
 
-    self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
-        security_enabled = False,
-        hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',
-        keytab = UnknownConfigurationMock(),
-        source = '/etc/zeppelin/conf/interpreter.json',
-        default_fs = 'hdfs://c6401.ambari.apache.org:8020',
-        replace_existing_files = True,
-        hdfs_resource_ignore_file = '/var/lib/ambari-agent/data/.hdfs_resource_ignore',
-        hdfs_site = {u'a': u'b'},
-        kinit_path_local = '/usr/bin/kinit',
-        principal_name = UnknownConfigurationMock(),
-        user = 'zeppelin',
-        owner = 'zeppelin',
-        group = 'zeppelin',
-        hadoop_conf_dir = '/usr/hdp/2.5.0.0-1235/hadoop/conf',
-        type = 'file',
-        action = ['create_on_execute'],
-    )
+    self.assertResourceCalled('HdfsResource',
+                              '/user/zeppelin/hdfs:///user/zeppelin/conf',
+                              security_enabled=False,
+                              hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+                              keytab=UnknownConfigurationMock(),
+                              default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              recursive_chown=True,
+                              recursive_chmod=True,
+                              hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              hdfs_site={u'a': u'b'},
+                              kinit_path_local='/usr/bin/kinit',
+                              principal_name=UnknownConfigurationMock(),
+                              user='hdfs',
+                              owner='zeppelin',
+                              hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+                              type='directory',
+                              action=['create_on_execute'],
+                              )
+
+    self.assertResourceCalled('HdfsResource',
+                              '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
+                              security_enabled=False,
+                              hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
+                              keytab=UnknownConfigurationMock(),
+                              source='/etc/zeppelin/conf/interpreter.json',
+                              default_fs='hdfs://c6401.ambari.apache.org:8020',
+                              replace_existing_files=True,
+                              hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              hdfs_site={u'a': u'b'},
+                              kinit_path_local='/usr/bin/kinit',
+                              principal_name=UnknownConfigurationMock(),
+                              recursive_chmod=True,
+                              recursive_chown=True,
+                              user='hdfs',
+                              owner='zeppelin',
+                              hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
+                              type='file',
+                              action=['create_on_execute'],
+                              )
 
     self.assertResourceCalled('File', '/etc/zeppelin/conf/interpreter.json',
                               content=interpreter_json_generated.template_after_kerberos,
@@ -362,6 +439,24 @@ class TestZeppelin070(RMFTestCase):
                               group='zeppelin',
                               )
 
+    self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf',
+                              security_enabled = False,
+                              hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',
+                              keytab = UnknownConfigurationMock(),
+                              default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+                              hdfs_resource_ignore_file = '/var/lib/ambari-agent/data/.hdfs_resource_ignore',
+                              hdfs_site = {u'a': u'b'},
+                              kinit_path_local = '/usr/bin/kinit',
+                              principal_name = UnknownConfigurationMock(),
+                              recursive_chown=True,
+                              recursive_chmod=True,
+                              user = 'hdfs',
+                              owner = 'zeppelin',
+                              hadoop_conf_dir = '/usr/hdp/2.5.0.0-1235/hadoop/conf',
+                              type = 'directory',
+                              action = ['create_on_execute'],
+                              )
+
     self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
         security_enabled = False,
         hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',
@@ -373,9 +468,10 @@ class TestZeppelin070(RMFTestCase):
         kinit_path_local = '/usr/bin/kinit',
         principal_name = UnknownConfigurationMock(),
         replace_existing_files = True,
-        user = 'zeppelin',
+        recursive_chown=True,
+        recursive_chmod=True,
+        user = 'hdfs',
         owner = 'zeppelin',
-        group = 'zeppelin',
         hadoop_conf_dir = '/usr/hdp/2.5.0.0-1235/hadoop/conf',
         type = 'file',
         action = ['create_on_execute'],


Mime
View raw message