ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject [41/50] [abbrv] ambari git commit: AMBARI-22212.Alter logic for storing interpreter.json(Prabhjyot Singh via Venkata Sairam)
Date Fri, 13 Oct 2017 15:11:00 GMT
AMBARI-22212.Alter logic for storing interpreter.json(Prabhjyot Singh via Venkata Sairam)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/be605cbf
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/be605cbf
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/be605cbf

Branch: refs/heads/branch-feature-AMBARI-20859
Commit: be605cbfdc977ff927df89fc772b8fbb1f589fce
Parents: 3c3b1b8
Author: Venkata Sairam <venkatasairam.lanka@gmail.com>
Authored: Thu Oct 12 17:17:07 2017 +0530
Committer: Venkata Sairam <venkatasairam.lanka@gmail.com>
Committed: Thu Oct 12 17:17:07 2017 +0530

----------------------------------------------------------------------
 .../ZEPPELIN/0.7.0/package/scripts/master.py    | 69 +++++++++-----------
 .../ZEPPELIN/0.7.0/package/scripts/params.py    |  5 ++
 .../stacks/2.6/ZEPPELIN/test_zeppelin_070.py    | 52 +++------------
 3 files changed, 44 insertions(+), 82 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/be605cbf/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
index eaa2cb9..a8b1b32 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/master.py
@@ -242,6 +242,17 @@ class Master(Script):
     if zeppelin_spark_dependencies and os.path.exists(zeppelin_spark_dependencies[0]):
       self.create_zeppelin_dir(params)
 
+    if params.conf_stored_in_hdfs:
+      if not self.is_path_exists_in_HDFS(self.get_zeppelin_conf_FS_directory(params), params.zeppelin_user):
+        # hdfs dfs -mkdir {zeppelin's conf directory}
+        params.HdfsResource(self.get_zeppelin_conf_FS_directory(params),
+                            type="directory",
+                            action="create_on_execute",
+                            owner=params.zeppelin_user,
+                            recursive_chown=True,
+                            recursive_chmod=True
+                            )
+
     # if first_setup:
     if not glob.glob(params.conf_dir + "/interpreter.json") and \
       not os.path.exists(params.conf_dir + "/interpreter.json"):
@@ -323,21 +334,16 @@ class Master(Script):
     import json
 
     interpreter_config = os.path.join(params.conf_dir, "interpreter.json")
-    if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config']
\
-      and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage']
== 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
-
-      if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
-        zeppelin_conf_fs = self.get_zeppelin_conf_FS(params)
-
-        if self.is_path_exists_in_HDFS(zeppelin_conf_fs, params.zeppelin_user):
-          # copy from hdfs to /etc/zeppelin/conf/interpreter.json
-          kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',None))
-          kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
-          shell.call(format("rm {interpreter_config};"
-                            "{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -get {zeppelin_conf_fs}
{interpreter_config}"),
-                     user=params.zeppelin_user)
-        else:
-          Logger.info(format("{zeppelin_conf_fs} does not exist. Skipping upload of DFS."))
+    if params.conf_stored_in_hdfs:
+      zeppelin_conf_fs = self.get_zeppelin_conf_FS(params)
+
+      if self.is_path_exists_in_HDFS(zeppelin_conf_fs, params.zeppelin_user):
+        # copy from hdfs to /etc/zeppelin/conf/interpreter.json
+        kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths',None))
+        kinit_if_needed = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};")
+        shell.call(format("rm {interpreter_config};"
+                          "{kinit_if_needed} hdfs --config {hadoop_conf_dir} dfs -get {zeppelin_conf_fs}
{interpreter_config}"),
+                   user=params.zeppelin_user)
 
     config_content = sudo.read_file(interpreter_config)
     config_data = json.loads(config_content)
@@ -353,28 +359,15 @@ class Master(Script):
          owner=params.zeppelin_user,
          content=json.dumps(config_data, indent=2))
 
-    if 'zeppelin.notebook.storage' in params.config['configurations']['zeppelin-config']
\
-      and params.config['configurations']['zeppelin-config']['zeppelin.notebook.storage']
== 'org.apache.zeppelin.notebook.repo.FileSystemNotebookRepo':
-
-      if 'zeppelin.config.fs.dir' in params.config['configurations']['zeppelin-config']:
-        if not self.is_path_exists_in_HDFS(self.get_zeppelin_conf_FS_directory(params), params.zeppelin_user):
-          # hdfs dfs -mkdir {zeppelin's conf directory}
-          params.HdfsResource(self.get_zeppelin_conf_FS_directory(params),
-                              type="directory",
-                              action="create_on_execute",
-                              owner=params.zeppelin_user,
-                              recursive_chown=True,
-                              recursive_chmod=True
-                          )
-
-        params.HdfsResource(self.get_zeppelin_conf_FS(params),
-                            type="file",
-                            action="create_on_execute",
-                            source=interpreter_config,
-                            owner=params.zeppelin_user,
-                            recursive_chown=True,
-                            recursive_chmod=True,
-                            replace_existing_files=True)
+    if params.conf_stored_in_hdfs:
+      params.HdfsResource(self.get_zeppelin_conf_FS(params),
+                          type="file",
+                          action="create_on_execute",
+                          source=interpreter_config,
+                          owner=params.zeppelin_user,
+                          recursive_chown=True,
+                          recursive_chmod=True,
+                          replace_existing_files=True)
 
   def update_kerberos_properties(self):
     import params
@@ -451,7 +444,7 @@ class Master(Script):
     hive_interactive_properties_key = 'hive_interactive'
     for setting_key in interpreter_settings.keys():
       interpreter = interpreter_settings[setting_key]
-      if interpreter['group'] == 'jdbc':
+      if interpreter['group'] == 'jdbc' and interpreter['name'] == 'jdbc':
         interpreter['dependencies'] = []
 
         if not params.hive_server_host and params.hive_server_interactive_hosts:

http://git-wip-us.apache.org/repos/asf/ambari/blob/be605cbf/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
index b7d3df0..e69037c 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.7.0/package/scripts/params.py
@@ -104,6 +104,11 @@ conf_dir = "/etc/zeppelin/conf"
 external_dependency_conf = "/etc/zeppelin/conf/external-dependency-conf"
 notebook_dir = os.path.join(*[install_dir, zeppelin_dirname, 'notebook'])
 
+conf_stored_in_hdfs = False
+if 'zeppelin.config.fs.dir' in config['configurations']['zeppelin-config'] and \
+  not config['configurations']['zeppelin-config']['zeppelin.config.fs.dir'].startswith('file://'):
+  conf_stored_in_hdfs = True
+
 # zeppelin-env.sh
 zeppelin_env_content = config['configurations']['zeppelin-env']['zeppelin_env_content']
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/be605cbf/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
index 3adb94c..e5d0240 100644
--- a/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
+++ b/ambari-server/src/test/python/stacks/2.6/ZEPPELIN/test_zeppelin_070.py
@@ -341,12 +341,6 @@ class TestZeppelin070(RMFTestCase):
                               security_enabled=False,
                               )
 
-    self.assertResourceCalled('File', '/etc/zeppelin/conf/interpreter.json',
-                          content=interpreter_json_generated.template_after_base,
-                          owner='zeppelin',
-                          group='zeppelin',
-                          )
-
     self.assertResourceCalled('HdfsResource',
                               '/user/zeppelin/hdfs:///user/zeppelin/conf',
                               security_enabled=False,
@@ -359,13 +353,20 @@ class TestZeppelin070(RMFTestCase):
                               principal_name=UnknownConfigurationMock(),
                               recursive_chown=True,
                               recursive_chmod=True,
-                              owner='zeppelin',
                               user='hdfs',
+                              owner='zeppelin',
                               hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
                               type='directory',
                               action=['create_on_execute'],
                               )
 
+    self.assertResourceCalled('File', '/etc/zeppelin/conf/interpreter.json',
+                          content=interpreter_json_generated.template_after_base,
+                          owner='zeppelin',
+                          group='zeppelin',
+                          )
+
+
     self.assertResourceCalled('HdfsResource',
                               '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
                               security_enabled=False,
@@ -394,25 +395,6 @@ class TestZeppelin070(RMFTestCase):
                               )
 
     self.assertResourceCalled('HdfsResource',
-                              '/user/zeppelin/hdfs:///user/zeppelin/conf',
-                              security_enabled=False,
-                              hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
-                              keytab=UnknownConfigurationMock(),
-                              default_fs='hdfs://c6401.ambari.apache.org:8020',
-                              recursive_chown=True,
-                              recursive_chmod=True,
-                              hdfs_resource_ignore_file='/var/lib/ambari-agent/data/.hdfs_resource_ignore',
-                              hdfs_site={u'a': u'b'},
-                              kinit_path_local='/usr/bin/kinit',
-                              principal_name=UnknownConfigurationMock(),
-                              user='hdfs',
-                              owner='zeppelin',
-                              hadoop_conf_dir='/usr/hdp/2.5.0.0-1235/hadoop/conf',
-                              type='directory',
-                              action=['create_on_execute'],
-                              )
-
-    self.assertResourceCalled('HdfsResource',
                               '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
                               security_enabled=False,
                               hadoop_bin_dir='/usr/hdp/2.5.0.0-1235/hadoop/bin',
@@ -439,24 +421,6 @@ class TestZeppelin070(RMFTestCase):
                               group='zeppelin',
                               )
 
-    self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf',
-                              security_enabled = False,
-                              hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',
-                              keytab = UnknownConfigurationMock(),
-                              default_fs = 'hdfs://c6401.ambari.apache.org:8020',
-                              hdfs_resource_ignore_file = '/var/lib/ambari-agent/data/.hdfs_resource_ignore',
-                              hdfs_site = {u'a': u'b'},
-                              kinit_path_local = '/usr/bin/kinit',
-                              principal_name = UnknownConfigurationMock(),
-                              recursive_chown=True,
-                              recursive_chmod=True,
-                              user = 'hdfs',
-                              owner = 'zeppelin',
-                              hadoop_conf_dir = '/usr/hdp/2.5.0.0-1235/hadoop/conf',
-                              type = 'directory',
-                              action = ['create_on_execute'],
-                              )
-
     self.assertResourceCalled('HdfsResource', '/user/zeppelin/hdfs:///user/zeppelin/conf/interpreter.json',
         security_enabled = False,
         hadoop_bin_dir = '/usr/hdp/2.5.0.0-1235/hadoop/bin',


Mime
View raw message