spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From wenc...@apache.org
Subject spark git commit: [SPARK-11753][SQL][TEST-HADOOP2.2] Make allowNonNumericNumbers option work
Date Tue, 24 May 2016 16:43:45 GMT
Repository: spark
Updated Branches:
  refs/heads/master 6075f5b4d -> c24b6b679


[SPARK-11753][SQL][TEST-HADOOP2.2] Make allowNonNumericNumbers option work

## What changes were proposed in this pull request?

Jackson suppprts `allowNonNumericNumbers` option to parse non-standard non-numeric numbers
such as "NaN", "Infinity", "INF".  Currently used Jackson version (2.5.3) doesn't support
it all. This patch upgrades the library and make the two ignored tests in `JsonParsingOptionsSuite`
passed.

## How was this patch tested?

`JsonParsingOptionsSuite`.

Author: Liang-Chi Hsieh <simonh@tw.ibm.com>
Author: Liang-Chi Hsieh <viirya@appier.com>

Closes #9759 from viirya/fix-json-nonnumric.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c24b6b67
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c24b6b67
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c24b6b67

Branch: refs/heads/master
Commit: c24b6b679c3efa053f7de19be73eb36dc70d9930
Parents: 6075f5b
Author: Liang-Chi Hsieh <simonh@tw.ibm.com>
Authored: Tue May 24 09:43:39 2016 -0700
Committer: Wenchen Fan <wenchen@databricks.com>
Committed: Tue May 24 09:43:39 2016 -0700

----------------------------------------------------------------------
 dev/deps/spark-deps-hadoop-2.2                  | 11 ++--
 dev/deps/spark-deps-hadoop-2.3                  | 11 ++--
 dev/deps/spark-deps-hadoop-2.4                  | 11 ++--
 dev/deps/spark-deps-hadoop-2.6                  | 11 ++--
 dev/deps/spark-deps-hadoop-2.7                  | 11 ++--
 pom.xml                                         |  8 ++-
 python/pyspark/sql/readwriter.py                |  3 +
 .../org/apache/spark/sql/DataFrameReader.scala  |  2 +
 .../datasources/json/JacksonParser.scala        | 28 ++++++----
 .../json/JsonParsingOptionsSuite.scala          | 59 +++++++++++++++-----
 10 files changed, 102 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/dev/deps/spark-deps-hadoop-2.2
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2
index 91b333f..a9068da 100644
--- a/dev/deps/spark-deps-hadoop-2.2
+++ b/dev/deps/spark-deps-hadoop-2.2
@@ -72,12 +72,13 @@ hk2-utils-2.4.0-b34.jar
 httpclient-4.5.2.jar
 httpcore-4.4.4.jar
 ivy-2.4.0.jar
-jackson-annotations-2.5.3.jar
-jackson-core-2.5.3.jar
+jackson-annotations-2.7.3.jar
+jackson-core-2.7.3.jar
 jackson-core-asl-1.9.13.jar
-jackson-databind-2.5.3.jar
+jackson-databind-2.7.3.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.11-2.5.3.jar
+jackson-module-paranamer-2.7.3.jar
+jackson-module-scala_2.11-2.7.3.jar
 janino-2.7.8.jar
 javassist-3.18.1-GA.jar
 javax.annotation-api-1.2.jar
@@ -127,7 +128,7 @@ objenesis-2.1.jar
 opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
-paranamer-2.6.jar
+paranamer-2.8.jar
 parquet-column-1.7.0.jar
 parquet-common-1.7.0.jar
 parquet-encoding-1.7.0.jar

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/dev/deps/spark-deps-hadoop-2.3
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3
index 61ed4c0..7e60a31 100644
--- a/dev/deps/spark-deps-hadoop-2.3
+++ b/dev/deps/spark-deps-hadoop-2.3
@@ -74,12 +74,13 @@ hk2-utils-2.4.0-b34.jar
 httpclient-4.5.2.jar
 httpcore-4.4.4.jar
 ivy-2.4.0.jar
-jackson-annotations-2.5.3.jar
-jackson-core-2.5.3.jar
+jackson-annotations-2.7.3.jar
+jackson-core-2.7.3.jar
 jackson-core-asl-1.9.13.jar
-jackson-databind-2.5.3.jar
+jackson-databind-2.7.3.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.11-2.5.3.jar
+jackson-module-paranamer-2.7.3.jar
+jackson-module-scala_2.11-2.7.3.jar
 janino-2.7.8.jar
 java-xmlbuilder-1.0.jar
 javassist-3.18.1-GA.jar
@@ -134,7 +135,7 @@ objenesis-2.1.jar
 opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
-paranamer-2.6.jar
+paranamer-2.8.jar
 parquet-column-1.7.0.jar
 parquet-common-1.7.0.jar
 parquet-encoding-1.7.0.jar

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/dev/deps/spark-deps-hadoop-2.4
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4
index fb01492..70d33b4 100644
--- a/dev/deps/spark-deps-hadoop-2.4
+++ b/dev/deps/spark-deps-hadoop-2.4
@@ -74,12 +74,13 @@ hk2-utils-2.4.0-b34.jar
 httpclient-4.5.2.jar
 httpcore-4.4.4.jar
 ivy-2.4.0.jar
-jackson-annotations-2.5.3.jar
-jackson-core-2.5.3.jar
+jackson-annotations-2.7.3.jar
+jackson-core-2.7.3.jar
 jackson-core-asl-1.9.13.jar
-jackson-databind-2.5.3.jar
+jackson-databind-2.7.3.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.11-2.5.3.jar
+jackson-module-paranamer-2.7.3.jar
+jackson-module-scala_2.11-2.7.3.jar
 janino-2.7.8.jar
 java-xmlbuilder-1.0.jar
 javassist-3.18.1-GA.jar
@@ -134,7 +135,7 @@ objenesis-2.1.jar
 opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
-paranamer-2.6.jar
+paranamer-2.8.jar
 parquet-column-1.7.0.jar
 parquet-common-1.7.0.jar
 parquet-encoding-1.7.0.jar

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/dev/deps/spark-deps-hadoop-2.6
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index 0baf4e8..a80f6bc 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -80,13 +80,14 @@ htrace-core-3.0.4.jar
 httpclient-4.5.2.jar
 httpcore-4.4.4.jar
 ivy-2.4.0.jar
-jackson-annotations-2.5.3.jar
-jackson-core-2.5.3.jar
+jackson-annotations-2.7.3.jar
+jackson-core-2.7.3.jar
 jackson-core-asl-1.9.13.jar
-jackson-databind-2.5.3.jar
+jackson-databind-2.7.3.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.11-2.5.3.jar
+jackson-module-paranamer-2.7.3.jar
+jackson-module-scala_2.11-2.7.3.jar
 jackson-xc-1.9.13.jar
 janino-2.7.8.jar
 java-xmlbuilder-1.0.jar
@@ -142,7 +143,7 @@ objenesis-2.1.jar
 opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
-paranamer-2.6.jar
+paranamer-2.8.jar
 parquet-column-1.7.0.jar
 parquet-common-1.7.0.jar
 parquet-encoding-1.7.0.jar

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/dev/deps/spark-deps-hadoop-2.7
----------------------------------------------------------------------
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 8be218c..c0b53f7 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -80,13 +80,14 @@ htrace-core-3.1.0-incubating.jar
 httpclient-4.5.2.jar
 httpcore-4.4.4.jar
 ivy-2.4.0.jar
-jackson-annotations-2.5.3.jar
-jackson-core-2.5.3.jar
+jackson-annotations-2.7.3.jar
+jackson-core-2.7.3.jar
 jackson-core-asl-1.9.13.jar
-jackson-databind-2.5.3.jar
+jackson-databind-2.7.3.jar
 jackson-jaxrs-1.9.13.jar
 jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.11-2.5.3.jar
+jackson-module-paranamer-2.7.3.jar
+jackson-module-scala_2.11-2.7.3.jar
 jackson-xc-1.9.13.jar
 janino-2.7.8.jar
 java-xmlbuilder-1.0.jar
@@ -143,7 +144,7 @@ objenesis-2.1.jar
 opencsv-2.3.jar
 oro-2.0.8.jar
 osgi-resource-locator-1.0.1.jar
-paranamer-2.6.jar
+paranamer-2.8.jar
 parquet-column-1.7.0.jar
 parquet-common-1.7.0.jar
 parquet-encoding-1.7.0.jar

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e778f77..f28aa14 100644
--- a/pom.xml
+++ b/pom.xml
@@ -160,7 +160,7 @@
     <jline.version>${scala.version}</jline.version>
     <jline.groupid>org.scala-lang</jline.groupid>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
-    <fasterxml.jackson.version>2.5.3</fasterxml.jackson.version>
+    <fasterxml.jackson.version>2.7.3</fasterxml.jackson.version>
     <snappy.version>1.1.2.4</snappy.version>
     <netlib.java.version>1.1.2</netlib.java.version>
     <calcite.version>1.2.0-incubating</calcite.version>
@@ -180,6 +180,7 @@
     <antlr4.version>4.5.2-1</antlr4.version>
     <jpam.version>1.1</jpam.version>
     <selenium.version>2.52.0</selenium.version>
+    <paranamer.version>2.8</paranamer.version>
 
     <test.java.home>${java.home}</test.java.home>
     <test.exclude.tags></test.exclude.tags>
@@ -1825,6 +1826,11 @@
         <artifactId>antlr4-runtime</artifactId>
         <version>${antlr4.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.thoughtworks.paranamer</groupId>
+        <artifactId>paranamer</artifactId>
+        <version>${paranamer.version}</version>
+      </dependency>
     </dependencies>
   </dependencyManagement>
 

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/python/pyspark/sql/readwriter.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index 855c9d6..6f788cf 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -193,6 +193,9 @@ class DataFrameReader(object):
                                         set, it uses the default value, ``true``.
         :param allowNumericLeadingZero: allows leading zeros in numbers (e.g. 00012). If
None is
                                         set, it uses the default value, ``false``.
+        :param allowNonNumericNumbers: allows using non-numeric numbers such as "NaN", "Infinity",
+                                       "-Infinity", "INF", "-INF", which are convertd to
floating
+                                       point numbers, ``true``.
         :param allowBackslashEscapingAnyCharacter: allows accepting quoting of all character
                                                    using backslash quoting mechanism. If
None is
                                                    set, it uses the default value, ``false``.

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index 57a2091..0fed917 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -293,6 +293,8 @@ class DataFrameReader private[sql](sparkSession: SparkSession) extends
Logging {
    * </li>
    * <li>`allowNumericLeadingZeros` (default `false`): allows leading zeros in numbers
    * (e.g. 00012)</li>
+   * <li>`allowNonNumericNumbers` (default `true`): allows using non-numeric numbers
such as "NaN",
+   * "Infinity", "-Infinity", "INF", "-INF", which are convertd to floating point numbers.</li>
    * <li>`allowBackslashEscapingAnyCharacter` (default `false`): allows accepting quoting
of all
    * character using backslash quoting mechanism</li>
    * <li>`mode` (default `PERMISSIVE`): allows a mode for dealing with corrupt records

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala
index aeee260..cafca32 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/json/JacksonParser.scala
@@ -129,13 +129,15 @@ object JacksonParser extends Logging {
       case (VALUE_STRING, FloatType) =>
         // Special case handling for NaN and Infinity.
         val value = parser.getText
-        val lowerCaseValue = value.toLowerCase()
-        if (lowerCaseValue.equals("nan") ||
-          lowerCaseValue.equals("infinity") ||
-          lowerCaseValue.equals("-infinity") ||
-          lowerCaseValue.equals("inf") ||
-          lowerCaseValue.equals("-inf")) {
+        if (value.equals("NaN") ||
+          value.equals("Infinity") ||
+          value.equals("+Infinity") ||
+          value.equals("-Infinity")) {
           value.toFloat
+        } else if (value.equals("+INF") || value.equals("INF")) {
+          Float.PositiveInfinity
+        } else if (value.equals("-INF")) {
+          Float.NegativeInfinity
         } else {
           throw new SparkSQLJsonProcessingException(s"Cannot parse $value as FloatType.")
         }
@@ -146,13 +148,15 @@ object JacksonParser extends Logging {
       case (VALUE_STRING, DoubleType) =>
         // Special case handling for NaN and Infinity.
         val value = parser.getText
-        val lowerCaseValue = value.toLowerCase()
-        if (lowerCaseValue.equals("nan") ||
-          lowerCaseValue.equals("infinity") ||
-          lowerCaseValue.equals("-infinity") ||
-          lowerCaseValue.equals("inf") ||
-          lowerCaseValue.equals("-inf")) {
+        if (value.equals("NaN") ||
+          value.equals("Infinity") ||
+          value.equals("+Infinity") ||
+          value.equals("-Infinity")) {
           value.toDouble
+        } else if (value.equals("+INF") || value.equals("INF")) {
+          Double.PositiveInfinity
+        } else if (value.equals("-INF")) {
+          Double.NegativeInfinity
         } else {
           throw new SparkSQLJsonProcessingException(s"Cannot parse $value as DoubleType.")
         }

http://git-wip-us.apache.org/repos/asf/spark/blob/c24b6b67/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
index c31dffe..2aab955 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonParsingOptionsSuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.execution.datasources.json
 
 import org.apache.spark.sql.QueryTest
 import org.apache.spark.sql.test.SharedSQLContext
+import org.apache.spark.sql.types.{DoubleType, StructField, StructType}
 
 /**
  * Test cases for various [[JSONOptions]].
@@ -93,23 +94,51 @@ class JsonParsingOptionsSuite extends QueryTest with SharedSQLContext
{
     assert(df.first().getLong(0) == 18)
   }
 
-  // The following two tests are not really working - need to look into Jackson's
-  // JsonParser.Feature.ALLOW_NON_NUMERIC_NUMBERS.
-  ignore("allowNonNumericNumbers off") {
-    val str = """{"age": NaN}"""
-    val rdd = spark.sparkContext.parallelize(Seq(str))
-    val df = spark.read.json(rdd)
-
-    assert(df.schema.head.name == "_corrupt_record")
+  test("allowNonNumericNumbers off") {
+    // non-quoted non-numeric numbers don't work if allowNonNumericNumbers is off.
+    var testCases: Seq[String] = Seq("""{"age": NaN}""", """{"age": Infinity}""",
+      """{"age": +Infinity}""", """{"age": -Infinity}""", """{"age": INF}""",
+      """{"age": +INF}""", """{"age": -INF}""")
+    testCases.foreach { str =>
+      val rdd = spark.sparkContext.parallelize(Seq(str))
+      val df = spark.read.option("allowNonNumericNumbers", "false").json(rdd)
+
+      assert(df.schema.head.name == "_corrupt_record")
+    }
+
+    // quoted non-numeric numbers should still work even allowNonNumericNumbers is off.
+    testCases = Seq("""{"age": "NaN"}""", """{"age": "Infinity"}""", """{"age": "+Infinity"}""",
+      """{"age": "-Infinity"}""", """{"age": "INF"}""", """{"age": "+INF"}""",
+      """{"age": "-INF"}""")
+    val tests: Seq[Double => Boolean] = Seq(_.isNaN, _.isPosInfinity, _.isPosInfinity,
+      _.isNegInfinity, _.isPosInfinity, _.isPosInfinity, _.isNegInfinity)
+    val schema = StructType(StructField("age", DoubleType, true) :: Nil)
+
+    testCases.zipWithIndex.foreach { case (str, idx) =>
+      val rdd = spark.sparkContext.parallelize(Seq(str))
+      val df = spark.read.option("allowNonNumericNumbers", "false").schema(schema).json(rdd)
+
+      assert(df.schema.head.name == "age")
+      assert(tests(idx)(df.first().getDouble(0)))
+    }
   }
 
-  ignore("allowNonNumericNumbers on") {
-    val str = """{"age": NaN}"""
-    val rdd = spark.sparkContext.parallelize(Seq(str))
-    val df = spark.read.option("allowNonNumericNumbers", "true").json(rdd)
-
-    assert(df.schema.head.name == "age")
-    assert(df.first().getDouble(0).isNaN)
+  test("allowNonNumericNumbers on") {
+    val testCases: Seq[String] = Seq("""{"age": NaN}""", """{"age": Infinity}""",
+      """{"age": +Infinity}""", """{"age": -Infinity}""", """{"age": +INF}""",
+      """{"age": -INF}""", """{"age": "NaN"}""", """{"age": "Infinity"}""",
+      """{"age": "-Infinity"}""")
+    val tests: Seq[Double => Boolean] = Seq(_.isNaN, _.isPosInfinity, _.isPosInfinity,
+      _.isNegInfinity, _.isPosInfinity, _.isNegInfinity, _.isNaN, _.isPosInfinity,
+      _.isNegInfinity, _.isPosInfinity, _.isNegInfinity)
+    val schema = StructType(StructField("age", DoubleType, true) :: Nil)
+    testCases.zipWithIndex.foreach { case (str, idx) =>
+      val rdd = spark.sparkContext.parallelize(Seq(str))
+      val df = spark.read.option("allowNonNumericNumbers", "true").schema(schema).json(rdd)
+
+      assert(df.schema.head.name == "age")
+      assert(tests(idx)(df.first().getDouble(0)))
+    }
   }
 
   test("allowBackslashEscapingAnyCharacter off") {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message