spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From joshro...@apache.org
Subject spark git commit: [SPARK-8903] Fix bug in cherry-pick of SPARK-8803
Date Wed, 08 Jul 2015 22:33:33 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-1.4 4df0f1b1b -> 3f6e6e0e2


[SPARK-8903] Fix bug in cherry-pick of SPARK-8803

This fixes a bug introduced in the cherry-pick of #7201 which led to a NullPointerException
when cross-tabulating a data set that contains null values.

Author: Josh Rosen <joshrosen@databricks.com>

Closes #7295 from JoshRosen/SPARK-8903 and squashes the following commits:

5489948 [Josh Rosen] [SPARK-8903] Fix bug in cherry-pick of SPARK-8803


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3f6e6e0e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3f6e6e0e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3f6e6e0e

Branch: refs/heads/branch-1.4
Commit: 3f6e6e0e2668832af1a54f5cb95e5a4537c7bc5a
Parents: 4df0f1b
Author: Josh Rosen <joshrosen@databricks.com>
Authored: Wed Jul 8 15:33:14 2015 -0700
Committer: Josh Rosen <joshrosen@databricks.com>
Committed: Wed Jul 8 15:33:14 2015 -0700

----------------------------------------------------------------------
 .../org/apache/spark/sql/execution/stat/StatFunctions.scala    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/3f6e6e0e/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
index 5a0c9a6..3c68028 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/stat/StatFunctions.scala
@@ -113,7 +113,7 @@ private[sql] object StatFunctions extends Logging {
       if (element == null) "null" else element.toString
     }
     // get the distinct values of column 2, so that we can make them the column names
-    val distinctCol2: Map[Any, Int] =
+    val distinctCol2: Map[String, Int] =
       counts.map(e => cleanElement(e.get(1))).distinct.zipWithIndex.toMap
     val columnSize = distinctCol2.size
     require(columnSize < 1e4, s"The number of distinct values for $col2, can't " +
@@ -128,7 +128,7 @@ private[sql] object StatFunctions extends Logging {
         countsRow.setLong(columnIndex + 1, row.getLong(2))
       }
       // the value of col1 is the first value, the rest are the counts
-      countsRow.setString(0, cleanElement(col1Item.toString))
+      countsRow.setString(0, cleanElement(col1Item))
       countsRow
     }.toSeq
     // Back ticks can't exist in DataFrame column names, therefore drop them. To be able
to accept
@@ -139,7 +139,7 @@ private[sql] object StatFunctions extends Logging {
     // In the map, the column names (._1) are not ordered by the index (._2). This was the
bug in
     // SPARK-8681. We need to explicitly sort by the column index and assign the column names.
     val headerNames = distinctCol2.toSeq.sortBy(_._2).map { r =>
-      StructField(cleanColumnName(r._1.toString), LongType)
+      StructField(cleanColumnName(r._1), LongType)
     }
     val schema = StructType(StructField(tableName, StringType) +: headerNames)
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message