zeppelin-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mina...@apache.org
Subject zeppelin git commit: [ZEPPELIN-1390] SparkInterpreter does not work for Spark2 version of HDP 2.5
Date Fri, 09 Sep 2016 11:08:40 GMT
Repository: zeppelin
Updated Branches:
  refs/heads/branch-0.6 5a270d43b -> 81ce3e641


[ZEPPELIN-1390] SparkInterpreter does not work for Spark2 version of HDP 2.5

### What is this PR for?
Spark2 version of HDP 2.5 is "2.0.0.2.5.0.0-1245". Currently, Zeppelin parses this version
to integer 2002500, which is higher than version "2.0.0" (i.e. 200 in integer). Therefore,
Zeppelin thinks it's not supported version and fail.

I fixed some codes to consider major, minor, and patch version. Now, version parsed using
only first 3 numbers (major, minor, patch) and parsed into a 5-digit integer. For example,
2.0.0 -> 20000, 1.6.2 -> 10602, 1.10.1 -> 11001. This makes HDP 2.5 compatible, and
also make it more robust than before. Since maybe Spark's minor version can be a 2-digit number
in the future

### What type of PR is it?
[Bug Fix]

### What is the Jira issue?
[ZEPPELIN-1390](https://issues.apache.org/jira/browse/ZEPPELIN-1390)

### How should this be tested?
I added SparkVersion test codes.

Author: Jun <i2r.jun@gmail.com>

Closes #1381 from tae-jun/ZEPPELIN-1390 and squashes the following commits:

bae5264 [Jun] Merge remote-tracking branch 'origin/master' into ZEPPELIN-1390
f5531ab [Jun] [ZEPPELIN-1390] Fix Python and R number versions
6ad8049 [Jun] [ZEPPELIN-1390] SparkInterpreter does not work for Spark2 version of HDP 2.5

(cherry picked from commit d005c7967a57a572cc6d6620185ac6aedbfc56e3)
Signed-off-by: Mina Lee <minalee@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/zeppelin/repo
Commit: http://git-wip-us.apache.org/repos/asf/zeppelin/commit/81ce3e64
Tree: http://git-wip-us.apache.org/repos/asf/zeppelin/tree/81ce3e64
Diff: http://git-wip-us.apache.org/repos/asf/zeppelin/diff/81ce3e64

Branch: refs/heads/branch-0.6
Commit: 81ce3e641d223252495eeeb527d2aab4332366ed
Parents: 5a270d4
Author: Jun <i2r.jun@gmail.com>
Authored: Sun Sep 4 10:53:32 2016 +0900
Committer: Mina Lee <minalee@apache.org>
Committed: Fri Sep 9 13:08:33 2016 +0200

----------------------------------------------------------------------
 .../main/java/org/apache/zeppelin/spark/SparkVersion.java | 10 ++++++++--
 spark/src/main/resources/R/zeppelin_sparkr.R              |  2 +-
 spark/src/main/resources/python/zeppelin_pyspark.py       |  6 +++---
 .../java/org/apache/zeppelin/spark/SparkVersionTest.java  |  9 +++++++--
 4 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81ce3e64/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
----------------------------------------------------------------------
diff --git a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
index 17f2de7..c8a7c79 100644
--- a/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
+++ b/spark/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
@@ -52,13 +52,19 @@ public class SparkVersion {
       if (pos > 0) {
         numberPart = versionString.substring(0, pos);
       }
-      version = Integer.parseInt(numberPart.replaceAll("\\.", ""));
+
+      String versions[] = numberPart.split("\\.");
+      int major = Integer.parseInt(versions[0]);
+      int minor = Integer.parseInt(versions[1]);
+      int patch = Integer.parseInt(versions[2]);
+      // version is always 5 digits. (e.g. 2.0.0 -> 20000, 1.6.2 -> 10602)
+      version = Integer.parseInt(String.format("%d%02d%02d", major, minor, patch));
     } catch (Exception e) {
       logger.error("Can not recognize Spark version " + versionString +
           ". Assume it's a future release", e);
 
       // assume it is future release
-      version = 999;
+      version = 99999;
     }
   }
 

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81ce3e64/spark/src/main/resources/R/zeppelin_sparkr.R
----------------------------------------------------------------------
diff --git a/spark/src/main/resources/R/zeppelin_sparkr.R b/spark/src/main/resources/R/zeppelin_sparkr.R
index d951774..8e66421 100644
--- a/spark/src/main/resources/R/zeppelin_sparkr.R
+++ b/spark/src/main/resources/R/zeppelin_sparkr.R
@@ -42,7 +42,7 @@ assign(".scStartTime", as.integer(Sys.time()), envir = SparkR:::.sparkREnv)
 # setup spark env
 assign(".sc", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext", "getSparkContext"),
envir = SparkR:::.sparkREnv)
 assign("sc", get(".sc", envir = SparkR:::.sparkREnv), envir=.GlobalEnv)
-if (version >= 200) {
+if (version >= 20000) {
   assign(".sparkRsession", SparkR:::callJStatic("org.apache.zeppelin.spark.ZeppelinRContext",
"getSparkSession"), envir = SparkR:::.sparkREnv)
   assign("spark", get(".sparkRsession", envir = SparkR:::.sparkREnv), envir = .GlobalEnv)
 }

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81ce3e64/spark/src/main/resources/python/zeppelin_pyspark.py
----------------------------------------------------------------------
diff --git a/spark/src/main/resources/python/zeppelin_pyspark.py b/spark/src/main/resources/python/zeppelin_pyspark.py
index 9a40556..3e6535f 100644
--- a/spark/src/main/resources/python/zeppelin_pyspark.py
+++ b/spark/src/main/resources/python/zeppelin_pyspark.py
@@ -107,9 +107,9 @@ class PyZeppelinContext(dict):
 
 
 class SparkVersion(object):
-  SPARK_1_4_0 = 140
-  SPARK_1_3_0 = 130
-  SPARK_2_0_0 = 200
+  SPARK_1_4_0 = 10400
+  SPARK_1_3_0 = 10300
+  SPARK_2_0_0 = 20000
 
   def __init__(self, versionNumber):
     self.version = versionNumber

http://git-wip-us.apache.org/repos/asf/zeppelin/blob/81ce3e64/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
----------------------------------------------------------------------
diff --git a/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java b/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
index 5783c1e..3dc8f4e 100644
--- a/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
+++ b/spark/src/test/java/org/apache/zeppelin/spark/SparkVersionTest.java
@@ -24,7 +24,7 @@ public class SparkVersionTest {
 
   @Test
   public void testUnknownSparkVersion() {
-    assertEquals(999, SparkVersion.fromVersionString("DEV-10.10").toNumber());
+    assertEquals(99999, SparkVersion.fromVersionString("DEV-10.10").toNumber());
   }
 
   @Test
@@ -33,6 +33,8 @@ public class SparkVersionTest {
     assertFalse(SparkVersion.fromVersionString("1.5.9").isUnsupportedVersion());
     assertTrue(SparkVersion.fromVersionString("0.9.0").isUnsupportedVersion());
     assertTrue(SparkVersion.UNSUPPORTED_FUTURE_VERSION.isUnsupportedVersion());
+    // should support spark2 version of HDP 2.5
+    assertFalse(SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245").isUnsupportedVersion());
   }
 
   @Test
@@ -40,6 +42,9 @@ public class SparkVersionTest {
     // test equals
     assertEquals(SparkVersion.SPARK_1_2_0, SparkVersion.fromVersionString("1.2.0"));
     assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT"));
+    assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT"));
+    // test spark2 version of HDP 2.5
+    assertEquals(SparkVersion.SPARK_2_0_0, SparkVersion.fromVersionString("2.0.0.2.5.0.0-1245"));
 
     // test newer than
     assertFalse(SparkVersion.SPARK_1_2_0.newerThan(SparkVersion.SPARK_1_2_0));
@@ -60,7 +65,7 @@ public class SparkVersionTest {
     assertTrue(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_3_0));
 
     // conversion
-    assertEquals(120, SparkVersion.SPARK_1_2_0.toNumber());
+    assertEquals(10200, SparkVersion.SPARK_1_2_0.toNumber());
     assertEquals("1.2.0", SparkVersion.SPARK_1_2_0.toString());
   }
 }


Mime
View raw message