spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dav...@apache.org
Subject spark git commit: Revert "[SPARK-15392][SQL] fix default value of size estimation of logical plan"
Date Wed, 18 May 2016 23:02:59 GMT
Repository: spark
Updated Branches:
  refs/heads/master fc29b896d -> 84b23453d


Revert "[SPARK-15392][SQL] fix default value of size estimation of logical plan"

This reverts commit fc29b896dae08b957ed15fa681b46162600a4050.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/84b23453
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/84b23453
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/84b23453

Branch: refs/heads/master
Commit: 84b23453ddb0a97e3d81306de0a5dcb64f88bdd0
Parents: fc29b89
Author: Davies Liu <davies.liu@gmail.com>
Authored: Wed May 18 16:02:52 2016 -0700
Committer: Davies Liu <davies.liu@gmail.com>
Committed: Wed May 18 16:02:52 2016 -0700

----------------------------------------------------------------------
 .../main/scala/org/apache/spark/sql/internal/SQLConf.scala  | 3 ++-
 .../test/scala/org/apache/spark/sql/DataFrameSuite.scala    | 9 ---------
 2 files changed, 2 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/84b23453/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index a7f4613..7933d12 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -605,7 +605,8 @@ private[sql] class SQLConf extends Serializable with CatalystConf with
Logging {
 
   def enableRadixSort: Boolean = getConf(RADIX_SORT_ENABLED)
 
-  def defaultSizeInBytes: Long = getConf(DEFAULT_SIZE_IN_BYTES, Long.MaxValue)
+  def defaultSizeInBytes: Long =
+    getConf(DEFAULT_SIZE_IN_BYTES, autoBroadcastJoinThreshold + 1L)
 
   def isParquetBinaryAsString: Boolean = getConf(PARQUET_BINARY_AS_STRING)
 

http://git-wip-us.apache.org/repos/asf/spark/blob/84b23453/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index df029e4..f573abf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -1476,13 +1476,4 @@ class DataFrameSuite extends QueryTest with SharedSQLContext {
       getMessage()
     assert(e1.startsWith("Path does not exist"))
   }
-
-  test("SPARK-15392: DataFrame created from RDD should not be broadcasted") {
-    val rdd = sparkContext.range(1, 100).map(i => Row(i, i))
-    val df = spark.createDataFrame(rdd, new StructType().add("a", LongType).add("b", LongType))
-    assert(df.queryExecution.analyzed.statistics.sizeInBytes >
-      spark.wrapped.conf.autoBroadcastJoinThreshold)
-    assert(df.selectExpr("a").queryExecution.analyzed.statistics.sizeInBytes >
-      spark.wrapped.conf.autoBroadcastJoinThreshold)
-  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message