spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From andrewo...@apache.org
Subject spark git commit: Revert "[SPARK-4808] Removing minimum number of elements read before spill check"
Date Sun, 22 Feb 2015 17:45:00 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-1.3 eed7389cf -> 4186dd3dd


Revert "[SPARK-4808] Removing minimum number of elements read before spill check"

This reverts commit 0382dcc0a94f8e619fd11ec2cc0b18459a690c2b.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4186dd3d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4186dd3d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4186dd3d

Branch: refs/heads/branch-1.3
Commit: 4186dd3dd074a41b3a1d6a4279b683fb355da092
Parents: eed7389
Author: Andrew Or <andrew@databricks.com>
Authored: Sun Feb 22 09:44:52 2015 -0800
Committer: Andrew Or <andrew@databricks.com>
Committed: Sun Feb 22 09:44:52 2015 -0800

----------------------------------------------------------------------
 .../scala/org/apache/spark/util/collection/Spillable.scala     | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/4186dd3d/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
index 747ecf0..9f54312 100644
--- a/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
+++ b/core/src/main/scala/org/apache/spark/util/collection/Spillable.scala
@@ -42,6 +42,9 @@ private[spark] trait Spillable[C] extends Logging {
   // Memory manager that can be used to acquire/release memory
   private[this] val shuffleMemoryManager = SparkEnv.get.shuffleMemoryManager
 
+  // Threshold for `elementsRead` before we start tracking this collection's memory usage
+  private[this] val trackMemoryThreshold = 1000
+
   // Initial threshold for the size of a collection before we start tracking its memory usage
   // Exposed for testing
   private[this] val initialMemoryThreshold: Long =
@@ -69,7 +72,8 @@ private[spark] trait Spillable[C] extends Logging {
    * @return true if `collection` was spilled to disk; false otherwise
    */
   protected def maybeSpill(collection: C, currentMemory: Long): Boolean = {
-    if (elementsRead % 32 == 0 && currentMemory >= myMemoryThreshold) {
+    if (elementsRead > trackMemoryThreshold && elementsRead % 32 == 0 &&
+        currentMemory >= myMemoryThreshold) {
       // Claim up to double our current memory from the shuffle memory pool
       val amountToRequest = 2 * currentMemory - myMemoryThreshold
       val granted = shuffleMemoryManager.tryToAcquire(amountToRequest)


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message