spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From pwend...@apache.org
Subject [4/7] git commit: Enable external sorting by default
Date Tue, 14 Jan 2014 06:32:46 GMT
Enable external sorting by default


Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/69c9aebe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/69c9aebe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/69c9aebe

Branch: refs/heads/master
Commit: 69c9aebed0dfd90e0a1c4d48cd24ea7ddd7624fa
Parents: 8d40e72
Author: Andrew Or <andrewor14@gmail.com>
Authored: Sun Jan 12 22:43:01 2014 -0800
Committer: Andrew Or <andrewor14@gmail.com>
Committed: Sun Jan 12 22:43:01 2014 -0800

----------------------------------------------------------------------
 core/src/main/scala/org/apache/spark/Aggregator.scala       | 2 +-
 core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/69c9aebe/core/src/main/scala/org/apache/spark/Aggregator.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/Aggregator.scala b/core/src/main/scala/org/apache/spark/Aggregator.scala
index c46b7bd..5fd90d0 100644
--- a/core/src/main/scala/org/apache/spark/Aggregator.scala
+++ b/core/src/main/scala/org/apache/spark/Aggregator.scala
@@ -32,7 +32,7 @@ case class Aggregator[K, V, C] (
     mergeCombiners: (C, C) => C) {
 
   private val sparkConf = SparkEnv.get.conf
-  private val externalSorting = sparkConf.getBoolean("spark.shuffle.externalSorting", false)
+  private val externalSorting = sparkConf.getBoolean("spark.shuffle.externalSorting", true)
 
   def combineValuesByKey(iter: Iterator[_ <: Product2[K, V]], context: TaskContext) :
Iterator[(K, C)] = {
     if (!externalSorting) {

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/69c9aebe/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala b/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala
index 34e8341..656c3ef 100644
--- a/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/CoGroupedRDD.scala
@@ -107,7 +107,7 @@ class CoGroupedRDD[K](@transient var rdds: Seq[RDD[_ <: Product2[K,
_]]], part:
 
   override def compute(s: Partition, context: TaskContext): Iterator[(K, CoGroupCombiner)]
= {
 
-    val externalSorting = sparkConf.getBoolean("spark.shuffle.externalSorting", false)
+    val externalSorting = sparkConf.getBoolean("spark.shuffle.externalSorting", true)
     val split = s.asInstanceOf[CoGroupPartition]
     val numRdds = split.deps.size
 


Mime
View raw message