ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dbhowm...@apache.org
Subject ambari git commit: AMBARI-17321 Zeppelin: Kerberos configurations and latest interpreter config missing. (Renjith Kamath via dipayanb)
Date Tue, 21 Jun 2016 19:04:25 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 5e243959b -> 9b034b234


AMBARI-17321 Zeppelin: Kerberos configurations and latest interpreter config missing. (Renjith
Kamath via dipayanb)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/9b034b23
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/9b034b23
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/9b034b23

Branch: refs/heads/trunk
Commit: 9b034b23420fb46ffef1ecdd287d2576548f49b4
Parents: 5e24395
Author: Dipayan Bhowmick <dipayan.bhowmick@gmail.com>
Authored: Wed Jun 22 00:33:39 2016 +0530
Committer: Dipayan Bhowmick <dipayan.bhowmick@gmail.com>
Committed: Wed Jun 22 00:34:10 2016 +0530

----------------------------------------------------------------------
 .../0.6.0.2.5/configuration/zeppelin-env.xml    | 17 +++++++++
 .../ZEPPELIN/0.6.0.2.5/kerberos.json            | 29 ++++++++++++--
 .../ZEPPELIN/0.6.0.2.5/metainfo.xml             | 16 ++++++++
 .../0.6.0.2.5/package/scripts/master.py         | 40 ++++++++++++++++----
 .../0.6.0.2.5/package/scripts/params.py         | 12 +++++-
 .../0.6.0.2.5/package/scripts/service_check.py  |  4 ++
 6 files changed, 105 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/9b034b23/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
index a912272..844ae98 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
@@ -221,4 +221,21 @@ log4j.appender.dailyfile.layout.ConversionPattern=%5p [%d] ({%t} %F[%M]:%L)
- %m
     </description>
     <on-ambari-upgrade add="true"/>
   </property>
+
+  <property>
+    <name>zeppelin.server.kerberos.principal</name>
+    <value>none</value>
+    <description>
+      Kerberos principal name for the Zeppelin.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
+  <property>
+    <name>zeppelin.server.kerberos.keytab</name>
+    <value>none</value>
+    <description>
+      Location of the kerberos keytab file for the Zeppelin.
+    </description>
+    <on-ambari-upgrade add="true"/>
+  </property>
 </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b034b23/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json
index f69ae45..3dcc67c 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/kerberos.json
@@ -5,13 +5,36 @@
       "identities": [
         {
           "name": "/smokeuser"
+        },
+        {
+          "name": "zeppelin_user",
+          "principal": {
+            "value": "zeppelin/${cluster_name}@${realm}",
+            "type" : "user",
+            "configuration": "zeppelin-env/zeppelin.server.kerberos.principal",
+            "local_username" : "zeppelin"
+          },
+          "keytab": {
+            "file": "${keytab_dir}/zeppelin.server.kerberos.keytab",
+            "owner": {
+              "name": "zeppelin",
+              "access": "r"
+            },
+            "group": {
+              "name": "${cluster-env/user_group}",
+              "access": ""
+            },
+            "configuration": "zeppelin-env/zeppelin.server.kerberos.keytab"
+          }
         }
       ],
-      "components": [
+      "configurations": [
         {
-          "name": "ZEPPELIN_MASTER"
+          "zeppelin-env": {
+            "zeppelin.kerberos.enabled": "true"
+          }
         }
       ]
     }
   ]
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b034b23/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
index f609156..c40a593 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/metainfo.xml
@@ -37,6 +37,22 @@ limitations under the License.
             <scriptType>PYTHON</scriptType>
             <timeout>10000</timeout>
           </commandScript>
+          <dependencies>
+            <dependency>
+              <name>SPARK/LIVY_SERVER</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+            <dependency>
+              <name>SPARK/SPARK_CLIENT</name>
+              <scope>host</scope>
+              <auto-deploy>
+                <enabled>true</enabled>
+              </auto-deploy>
+            </dependency>
+          </dependencies>
         </component>
       </components>
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b034b23/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index 22d66b0..f2899d0 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -144,14 +144,27 @@ class Master(Script):
   def start(self, env):
     import params
     import status_params
+    import time
     self.configure(env)
 
+    if params.security_enabled:
+        spark_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};
")
+        Execute(spark_kinit_cmd, user=params.zeppelin_user)
+
     if glob.glob(
             params.zeppelin_dir + '/interpreter/spark/dep/zeppelin-spark-dependencies-*.jar')
and os.path.exists(
       glob.glob(params.zeppelin_dir + '/interpreter/spark/dep/zeppelin-spark-dependencies-*.jar')[0]):
       self.create_zeppelin_dir(params)
 
-    Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh start >> '
+    # if first_setup:
+    if not glob.glob(params.conf_dir + "/interpreter.json") and \
+      not os.path.exists(params.conf_dir + "/interpreter.json"):
+      Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh start >> '
+              + params.zeppelin_log_file, user=params.zeppelin_user)
+    time.sleep(20)
+    self.update_zeppelin_interpreter()
+
+    Execute(params.zeppelin_dir + '/bin/zeppelin-daemon.sh restart >> '
             + params.zeppelin_log_file, user=params.zeppelin_user)
     pidfile = glob.glob(status_params.zeppelin_pid_dir
                         + '/zeppelin-' + params.zeppelin_user + '*.pid')[0]
@@ -159,11 +172,6 @@ class Master(Script):
     contents = open(pidfile).read()
     Execute('echo pid is ' + contents, user=params.zeppelin_user)
 
-    # if first_setup:
-    import time
-    time.sleep(20)
-    self.update_zeppelin_interpreter()
-
   def status(self, env):
     import status_params
     env.set_params(status_params)
@@ -187,15 +195,31 @@ class Master(Script):
 
     for notebooks in interpreter_settings:
         notebook = interpreter_settings[notebooks]
-        if notebook['group'] == 'hive' and params.hive_server_host:
-            notebook['properties']['hive.hiveserver2.url'] = 'jdbc:hive2://' +\
+        if notebook['group'] == 'jdbc' and params.hive_server_host:
+            notebook['properties']['default.url'] = 'jdbc:hive2://' +\
                                                              params.hive_server_host +\
                                                              ':' + params.hive_server_port
+            notebook['properties']['default.driver'] = "org.apache.hive.jdbc.HiveDriver"
+            notebook['dependencies'] = []
+            notebook['dependencies'].append(
+              {"groupArtifactVersion": "org.apache.hive:hive-jdbc:2.0.1", "local": "false"})
+            notebook['dependencies'].append(
+              {"groupArtifactVersion": "org.apache.hadoop:hadoop-common:2.7.2", "local":
"false"})
         elif notebook['group'] == 'phoenix' and params.zookeeper_znode_parent \
                 and params.hbase_zookeeper_quorum:
             notebook['properties']['phoenix.jdbc.url'] = "jdbc:phoenix:" +\
                                                          params.hbase_zookeeper_quorum +
':' +\
                                                          params.zookeeper_znode_parent
+        elif notebook['group'] == 'livy' and params.livy_livyserver_host:
+            notebook['properties']['livy.spark.master'] = "yarn-cluster"
+            notebook['properties']['zeppelin.livy.principal'] = params.zeppelin_kerberos_principal
+            notebook['properties']['zeppelin.livy.keytab'] = params.zeppelin_kerberos_keytab
+            notebook['properties']['zeppelin.livy.url'] = "http://" + params.livy_livyserver_host
+\
+                                                          ":" + params.livy_livyserver_port
+        elif notebook['group'] == 'spark':
+            notebook['properties']['master'] = "yarn-cluster"
+            notebook['properties']['spark.yarn.principal'] = params.zeppelin_kerberos_principal
+            notebook['properties']['spark.yarn.keytab'] = params.zeppelin_kerberos_keytab
 
     interpreter_config_file = open(interpreter_config, "w+")
     interpreter_config_file.write(json.dumps(config_data, indent=2))

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b034b23/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
index e5ce9f2..770f8f5 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
@@ -21,16 +21,17 @@ limitations under the License.
 import functools
 import os
 import re
-
+from resource_management.libraries.functions import StackFeature
 from resource_management.libraries.functions import conf_select
 from resource_management.libraries.functions import get_kinit_path
 from resource_management.libraries.functions import stack_select
 from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.get_stack_version import get_stack_version
+from resource_management.libraries.functions.stack_features import check_stack_feature
 from resource_management.libraries.functions.version import format_stack_version
 from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions.format import format
 
 def get_port_from_url(address):
   if not (address is None):
@@ -123,6 +124,9 @@ if 'spark.yarn.queue' in config['configurations']['spark-defaults']:
 else:
   spark_queue = 'default'
 
+zeppelin_kerberos_keytab = config['configurations']['zeppelin-env']['zeppelin.server.kerberos.keytab']
+zeppelin_kerberos_principal = config['configurations']['zeppelin-env']['zeppelin.server.kerberos.principal']
+
 # e.g. 2.3
 stack_version_unformatted = config['hostLevelParams']['stack_version']
 
@@ -134,6 +138,10 @@ full_stack_version = default("/commandParams/version", None)
 
 spark_client_version = get_stack_version('spark-client')
 
+if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted):
+  livy_livyserver_host = str(default("/clusterHostInfo/livy_server_hosts", [])[0])
+  livy_livyserver_port = config['configurations']['livy-conf']['livy.server.port']
+
 hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
 security_enabled = config['configurations']['cluster-env']['security_enabled']
 hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']

http://git-wip-us.apache.org/repos/asf/ambari/blob/9b034b23/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py
index 978976f..4170e10 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/service_check.py
@@ -25,6 +25,10 @@ class ZeppelinServiceCheck(Script):
         import params
         env.set_params(params)
 
+        if params.security_enabled:
+          spark_kinit_cmd = format("{kinit_path_local} -kt {zeppelin_kerberos_keytab} {zeppelin_kerberos_principal};
")
+          Execute(spark_kinit_cmd, user=params.zeppelin_user)
+
         Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k {zeppelin_host}:{zeppelin_port}
| grep 200"),
                 tries = 10,
                 try_sleep=3,


Mime
View raw message