ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rkam...@apache.org
Subject [2/2] ambari git commit: AMBARI-19126 Spark 2.x interpreter confgurations in Zeppelin (r-kamath)
Date Thu, 15 Dec 2016 06:31:45 GMT
AMBARI-19126 Spark 2.x interpreter confgurations in Zeppelin (r-kamath)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b77b2fac
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b77b2fac
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b77b2fac

Branch: refs/heads/trunk
Commit: b77b2fac39bc06df1d5b6a59d1be4d6bdd8b0386
Parents: f936fd0
Author: Renjith Kamath <renjith.kamath@gmail.com>
Authored: Thu Dec 15 11:57:34 2016 +0530
Committer: Renjith Kamath <renjith.kamath@gmail.com>
Committed: Thu Dec 15 11:59:13 2016 +0530

----------------------------------------------------------------------
 .../0.6.0.2.5/configuration/zeppelin-env.xml    |   2 +-
 .../0.6.0.2.5/package/scripts/master.py         | 108 +++++++++++--------
 .../0.6.0.2.5/package/scripts/params.py         |   4 +-
 .../package/scripts/spark2_config_template.py   |  64 +++++++++++
 4 files changed, 133 insertions(+), 45 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b77b2fac/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
index c03d2dc..317ad73 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/configuration/zeppelin-env.xml
@@ -115,7 +115,7 @@ export ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory={{executor_mem}} -Dspark.exec
 ## defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit
 ##
 # (required) When it is defined, load it instead of Zeppelin embedded Spark libraries
-export SPARK_HOME={{spark_home}}
+#export SPARK_HOME={{spark_home}}
 
 # (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory
1G".
 # export SPARK_SUBMIT_OPTIONS

http://git-wip-us.apache.org/repos/asf/ambari/blob/b77b2fac/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
index e82b53c..9cc808b 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/master.py
@@ -237,40 +237,40 @@ class Master(Script):
     import params
     config_data = self.get_interpreter_settings()
     interpreter_settings = config_data['interpreterSettings']
-    for notebooks in interpreter_settings:
-      notebook = interpreter_settings[notebooks]
-      if notebook['group'] == 'livy' and params.livy_livyserver_host:
+    for interpreter_setting in interpreter_settings:
+      interpreter = interpreter_settings[interpreter_setting]
+      if interpreter['group'] == 'livy' and params.livy_livyserver_host:
         if params.zeppelin_kerberos_principal and params.zeppelin_kerberos_keytab and params.security_enabled:
-          notebook['properties']['zeppelin.livy.principal'] = params.zeppelin_kerberos_principal
-          notebook['properties']['zeppelin.livy.keytab'] = params.zeppelin_kerberos_keytab
+          interpreter['properties']['zeppelin.livy.principal'] = params.zeppelin_kerberos_principal
+          interpreter['properties']['zeppelin.livy.keytab'] = params.zeppelin_kerberos_keytab
         else:
-          notebook['properties']['zeppelin.livy.principal'] = ""
-          notebook['properties']['zeppelin.livy.keytab'] = ""
-      elif notebook['group'] == 'spark':
+          interpreter['properties']['zeppelin.livy.principal'] = ""
+          interpreter['properties']['zeppelin.livy.keytab'] = ""
+      elif interpreter['group'] == 'spark':
         if params.zeppelin_kerberos_principal and params.zeppelin_kerberos_keytab and params.security_enabled:
-          notebook['properties']['spark.yarn.principal'] = params.zeppelin_kerberos_principal
-          notebook['properties']['spark.yarn.keytab'] = params.zeppelin_kerberos_keytab
+          interpreter['properties']['spark.yarn.principal'] = params.zeppelin_kerberos_principal
+          interpreter['properties']['spark.yarn.keytab'] = params.zeppelin_kerberos_keytab
         else:
-          notebook['properties']['spark.yarn.principal'] = ""
-          notebook['properties']['spark.yarn.keytab'] = ""
-      elif notebook['group'] == 'jdbc':
+          interpreter['properties']['spark.yarn.principal'] = ""
+          interpreter['properties']['spark.yarn.keytab'] = ""
+      elif interpreter['group'] == 'jdbc':
         if params.zeppelin_kerberos_principal and params.zeppelin_kerberos_keytab and params.security_enabled:
-          notebook['properties']['zeppelin.jdbc.auth.type'] = "KERBEROS"
-          notebook['properties']['zeppelin.jdbc.principal'] = params.zeppelin_kerberos_principal
-          notebook['properties']['zeppelin.jdbc.keytab.location'] = params.zeppelin_kerberos_keytab
+          interpreter['properties']['zeppelin.jdbc.auth.type'] = "KERBEROS"
+          interpreter['properties']['zeppelin.jdbc.principal'] = params.zeppelin_kerberos_principal
+          interpreter['properties']['zeppelin.jdbc.keytab.location'] = params.zeppelin_kerberos_keytab
         else:
-          notebook['properties']['zeppelin.jdbc.auth.type'] = ""
-          notebook['properties']['zeppelin.jdbc.principal'] = ""
-          notebook['properties']['zeppelin.jdbc.keytab.location'] = ""
-      elif notebook['group'] == 'sh':
+          interpreter['properties']['zeppelin.jdbc.auth.type'] = ""
+          interpreter['properties']['zeppelin.jdbc.principal'] = ""
+          interpreter['properties']['zeppelin.jdbc.keytab.location'] = ""
+      elif interpreter['group'] == 'sh':
         if params.zeppelin_kerberos_principal and params.zeppelin_kerberos_keytab and params.security_enabled:
-          notebook['properties']['zeppelin.shell.auth.type'] = "KERBEROS"
-          notebook['properties']['zeppelin.shell.principal'] = params.zeppelin_kerberos_principal
-          notebook['properties']['zeppelin.shell.keytab.location'] = params.zeppelin_kerberos_keytab
+          interpreter['properties']['zeppelin.shell.auth.type'] = "KERBEROS"
+          interpreter['properties']['zeppelin.shell.principal'] = params.zeppelin_kerberos_principal
+          interpreter['properties']['zeppelin.shell.keytab.location'] = params.zeppelin_kerberos_keytab
         else:
-          notebook['properties']['zeppelin.shell.auth.type'] = ""
-          notebook['properties']['zeppelin.shell.principal'] = ""
-          notebook['properties']['zeppelin.shell.keytab.location'] = ""
+          interpreter['properties']['zeppelin.shell.auth.type'] = ""
+          interpreter['properties']['zeppelin.shell.principal'] = ""
+          interpreter['properties']['zeppelin.shell.keytab.location'] = ""
 
     self.set_interpreter_settings(config_data)
 
@@ -279,39 +279,57 @@ class Master(Script):
     config_data = self.get_interpreter_settings()
     interpreter_settings = config_data['interpreterSettings']
 
-    for notebooks in interpreter_settings:
-      notebook = interpreter_settings[notebooks]
-      if notebook['group'] == 'jdbc':
-        notebook['dependencies'] = []
+    if params.spark2_home:
+      spark2_config = self.get_spark2_interpreter_config()
+      config_id = spark2_config["id"]
+      interpreter_settings[config_id] = spark2_config
+
+    for interpreter_setting in interpreter_settings:
+      interpreter = interpreter_settings[interpreter_setting]
+      if interpreter['group'] == 'jdbc':
+        interpreter['dependencies'] = []
         if params.hive_server_host:
           if params.hive_server2_support_dynamic_service_discovery:
-            notebook['properties']['hive.url'] = 'jdbc:hive2://' + \
+            interpreter['properties']['hive.url'] = 'jdbc:hive2://' + \
                                                  params.hive_zookeeper_quorum + \
                                                  '/;' + 'serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=hiveserver2'
           else:
-            notebook['properties']['hive.url'] = 'jdbc:hive2://' + \
+            interpreter['properties']['hive.url'] = 'jdbc:hive2://' + \
                                                  params.hive_server_host + \
                                                      ':' + params.hive_server_port
-          notebook['dependencies'].append(
+          interpreter['dependencies'].append(
               {"groupArtifactVersion": "org.apache.hive:hive-jdbc:2.0.1", "local": "false"})
-          notebook['dependencies'].append(
+          interpreter['dependencies'].append(
               {"groupArtifactVersion": "org.apache.hadoop:hadoop-common:2.7.2", "local":
"false"})
-          notebook['dependencies'].append(
+          interpreter['dependencies'].append(
               {"groupArtifactVersion": "org.apache.hive.shims:hive-shims-0.23:2.1.0", "local":
"false"})
 
         if params.zookeeper_znode_parent \
                 and params.hbase_zookeeper_quorum:
-            notebook['properties']['phoenix.url'] = "jdbc:phoenix:" + \
+            interpreter['properties']['phoenix.url'] = "jdbc:phoenix:" + \
                                                     params.hbase_zookeeper_quorum + ':' +
\
                                                     params.zookeeper_znode_parent
-            notebook['dependencies'].append(
+            interpreter['dependencies'].append(
                 {"groupArtifactVersion": "org.apache.phoenix:phoenix-core:4.7.0-HBase-1.1",
"local": "false"})
-      elif notebook['group'] == 'livy' and params.livy_livyserver_host:
-        notebook['properties']['livy.spark.master'] = "yarn-cluster"
-        notebook['properties']['zeppelin.livy.url'] = "http://" + params.livy_livyserver_host
+\
+      elif interpreter['group'] == 'livy' and params.livy_livyserver_host:
+        interpreter['properties']['livy.spark.master'] = "yarn-cluster"
+        interpreter['properties']['zeppelin.livy.url'] = "http://" + params.livy_livyserver_host
+\
                                                       ":" + params.livy_livyserver_port
-      elif notebook['group'] == 'spark':
-        notebook['properties']['master'] = "yarn-client"
+
+      elif interpreter['group'] == 'spark' and interpreter['name'] == 'spark':
+        if params.spark_home:
+          interpreter['properties']['master'] = "yarn-client"
+          interpreter['properties']['SPARK_HOME'] = "/usr/hdp/current/spark-client/"
+        else:
+          interpreter['properties']['master'] = "local[*]"
+
+      elif interpreter['group'] == 'spark' and interpreter['name'] == 'spark2':
+        if params.spark2_home:
+          interpreter['properties']['master'] = "yarn-client"
+          interpreter['properties']['SPARK_HOME'] = "/usr/hdp/current/spark2-client/"
+        else:
+          interpreter['properties']['master'] = "local[*]"
+
     self.set_interpreter_settings(config_data)
 
   def check_zeppelin_server(self, retries=10):
@@ -334,5 +352,11 @@ class Master(Script):
     import params
     return glob.glob(params.zeppelin_dir + '/interpreter/spark/dep/zeppelin-spark-dependencies*.jar')
 
+  def get_spark2_interpreter_config(self):
+    import spark2_config_template
+    import json
+
+    return json.loads(spark2_config_template.template)
+
 if __name__ == "__main__":
   Master().execute()

http://git-wip-us.apache.org/repos/asf/ambari/blob/b77b2fac/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
index 5a3174f..5ee6147 100644
--- a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/params.py
@@ -67,9 +67,9 @@ spark_jar = format("{spark_jar_dir}/zeppelin-spark-0.5.5-SNAPSHOT.jar")
 setup_view = True
 temp_file = config['configurations']['zeppelin-env']['zeppelin.temp.file']
 
-spark_home = None
+spark_home = ""
 spark_version = None
-spark2_home = None
+spark2_home = ""
 spark2_version = None
 if 'spark-defaults' in config['configurations']:
   spark_home = os.path.join(stack_root, "current", 'spark-client')

http://git-wip-us.apache.org/repos/asf/ambari/blob/b77b2fac/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/spark2_config_template.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/spark2_config_template.py
b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/spark2_config_template.py
new file mode 100644
index 0000000..0e81542
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ZEPPELIN/0.6.0.2.5/package/scripts/spark2_config_template.py
@@ -0,0 +1,64 @@
+template = '''
+{
+  "id": "2C4U48MY3_spark2",
+  "name": "spark2",
+  "group": "spark",
+  "properties": {
+    "spark.executor.memory": "",
+    "args": "",
+    "zeppelin.spark.printREPLOutput": "true",
+    "spark.cores.max": "",
+    "zeppelin.dep.additionalRemoteRepository": "spark-packages,http://dl.bintray.com/spark-packages/maven,false;",
+    "zeppelin.spark.importImplicit": "true",
+    "zeppelin.spark.sql.stacktrace": "false",
+    "zeppelin.spark.concurrentSQL": "false",
+    "zeppelin.spark.useHiveContext": "true",
+    "zeppelin.pyspark.python": "python",
+    "zeppelin.dep.localrepo": "local-repo",
+    "zeppelin.R.knitr": "true",
+    "zeppelin.spark.maxResult": "1000",
+    "master": "local[*]",
+    "spark.app.name": "Zeppelin",
+    "zeppelin.R.image.width": "100%",
+    "zeppelin.R.render.options": "out.format \u003d \u0027html\u0027, comment \u003d NA,
echo \u003d FALSE, results \u003d \u0027asis\u0027, message \u003d F, warning \u003d F",
+    "zeppelin.R.cmd": "R"
+  },
+  "status": "READY",
+  "interpreterGroup": [
+    {
+      "name": "spark",
+      "class": "org.apache.zeppelin.spark.SparkInterpreter",
+      "defaultInterpreter": true
+    },
+    {
+      "name": "sql",
+      "class": "org.apache.zeppelin.spark.SparkSqlInterpreter",
+      "defaultInterpreter": false
+    },
+    {
+      "name": "dep",
+      "class": "org.apache.zeppelin.spark.DepInterpreter",
+      "defaultInterpreter": false
+    },
+    {
+      "name": "pyspark",
+      "class": "org.apache.zeppelin.spark.PySparkInterpreter",
+      "defaultInterpreter": false
+    },
+    {
+      "name": "r",
+      "class": "org.apache.zeppelin.spark.SparkRInterpreter",
+      "defaultInterpreter": false
+    }
+  ],
+  "dependencies": [],
+  "option": {
+    "remote": true,
+    "port": -1,
+    "perNoteSession": false,
+    "perNoteProcess": false,
+    "isExistingProcess": false,
+    "setPermission": false
+  }
+}
+'''
\ No newline at end of file


Mime
View raw message