spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From r...@apache.org
Subject spark git commit: [SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.
Date Fri, 03 Jul 2015 07:25:06 GMT
Repository: spark
Updated Branches:
  refs/heads/master 9b23e92c7 -> 2848f4da4


[SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.

"NaN" from string to double is already handled by Cast expression itself.

Author: Reynold Xin <rxin@databricks.com>

Closes #7206 from rxin/convertnans and squashes the following commits:

3d99c33 [Reynold Xin] [SPARK-8809][SQL] Remove ConvertNaNs analyzer rule.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2848f4da
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2848f4da
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2848f4da

Branch: refs/heads/master
Commit: 2848f4da47d5c395de93ab9960bd905edfbd3439
Parents: 9b23e92
Author: Reynold Xin <rxin@databricks.com>
Authored: Fri Jul 3 00:25:02 2015 -0700
Committer: Reynold Xin <rxin@databricks.com>
Committed: Fri Jul 3 00:25:02 2015 -0700

----------------------------------------------------------------------
 .../catalyst/analysis/HiveTypeCoercion.scala    | 33 --------------------
 1 file changed, 33 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/2848f4da/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
index 6006e7b..38eb832 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
@@ -35,7 +35,6 @@ object HiveTypeCoercion {
 
   val typeCoercionRules =
     PropagateTypes ::
-      ConvertNaNs ::
       InConversion ::
       WidenTypes ::
       PromoteStrings ::
@@ -149,38 +148,6 @@ object HiveTypeCoercion {
   }
 
   /**
-   * Converts string "NaN"s that are in binary operators with a NaN-able types (Float / Double)
to
-   * the appropriate numeric equivalent.
-   */
-  // TODO: remove this rule and make Cast handle Nan.
-  object ConvertNaNs extends Rule[LogicalPlan] {
-    private val StringNaN = Literal("NaN")
-
-    def apply(plan: LogicalPlan): LogicalPlan = plan transform {
-      case q: LogicalPlan => q transformExpressions {
-        // Skip nodes who's children have not been resolved yet.
-        case e if !e.childrenResolved => e
-
-        /* Double Conversions */
-        case b @ BinaryOperator(StringNaN, right @ DoubleType()) =>
-          b.makeCopy(Array(Literal(Double.NaN), right))
-        case b @ BinaryOperator(left @ DoubleType(), StringNaN) =>
-          b.makeCopy(Array(left, Literal(Double.NaN)))
-
-        /* Float Conversions */
-        case b @ BinaryOperator(StringNaN, right @ FloatType()) =>
-          b.makeCopy(Array(Literal(Float.NaN), right))
-        case b @ BinaryOperator(left @ FloatType(), StringNaN) =>
-          b.makeCopy(Array(left, Literal(Float.NaN)))
-
-        /* Use float NaN by default to avoid unnecessary type widening */
-        case b @ BinaryOperator(left @ StringNaN, StringNaN) =>
-          b.makeCopy(Array(left, Literal(Float.NaN)))
-      }
-    }
-  }
-
-  /**
    * Widens numeric types and converts strings to numbers when appropriate.
    *
    * Loosely based on rules from "Hadoop: The Definitive Guide" 2nd edition, by Tom White


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message