tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject [06/26] tez git commit: TEZ-2290. Scale memory for Default Sorter down to a max of 2047 MB if configured higher (rbalamohan)
Date Thu, 09 Apr 2015 20:33:02 GMT
TEZ-2290. Scale memory for Default Sorter down to a max of 2047 MB if configured higher (rbalamohan)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/936ff8de
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/936ff8de
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/936ff8de

Branch: refs/heads/TEZ-2003
Commit: 936ff8de0c02b0b1e5c94b8c31598c8d40a9d62d
Parents: c636dc2
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Thu Apr 9 06:41:13 2015 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Thu Apr 9 06:41:13 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                     |  1 +
 .../common/sort/impl/dflt/DefaultSorter.java    | 27 +++++++++++++++-----
 .../sort/impl/dflt/TestDefaultSorter.java       | 20 +++++++++++++++
 3 files changed, 41 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/936ff8de/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 45c1541..cce0919 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,7 @@ INCOMPATIBLE CHANGES
   TEZ-1993. Implement a pluggable InputSizeEstimator for grouping fairly
 
 ALL CHANGES:
+  TEZ-2290. Scale memory for Default Sorter down to a max of 2047 MB if configured higher.
   TEZ-2233. Allow EdgeProperty of an edge to be changed by VertexManager
   TEZ-2293. When running in "mr" mode, always use MR config settings.
   TEZ-2273. Tez UI: Support client side searching & sorting for dag tasks page

http://git-wip-us.apache.org/repos/asf/tez/blob/936ff8de/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
index a6db0c6..2cbb70a 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
@@ -29,6 +29,7 @@ import java.util.List;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.ReentrantLock;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.Lists;
 import org.apache.commons.lang.exception.ExceptionUtils;
 import org.slf4j.Logger;
@@ -128,13 +129,8 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable
{
     final float spillper = this.conf.getFloat(
         TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT,
         TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT_DEFAULT);
-    final int sortmb = (int) availableMemoryMb;
-    if (sortmb <= 0) {
-      throw new RuntimeException("InitialMemoryAssigned is <= 0: " + initialMemoryAvailable);
-    }
-    Preconditions.checkArgument(sortmb > 0 && sortmb <= 2047,
-        TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB
-            + " for DefaultSorter should be larger than 0 and less than or equal to 2047");
+    final int sortmb = computeSortBufferSize((int) availableMemoryMb);
+
     Preconditions.checkArgument(spillper <= (float) 1.0 && spillper > (float)
0.0,
         TezRuntimeConfiguration.TEZ_RUNTIME_SORT_SPILL_PERCENT
             + " should be greater than 0 and less than or equal to 1");
@@ -207,6 +203,23 @@ public class DefaultSorter extends ExternalSorter implements IndexedSortable
{
     }
   }
 
+  @VisibleForTesting
+  static int computeSortBufferSize(int availableMemoryMB) {
+
+    if (availableMemoryMB <= 0) {
+      throw new RuntimeException(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB +
+          "=" + availableMemoryMB + ". It should be > 0");
+    }
+
+    if (availableMemoryMB > 2047) {
+      LOG.warn("Scaling down " + TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB +
+          "=" + availableMemoryMB + " to 2047 (max sort buffer size supported for DefaultSorter)");
+    }
+
+    //cap sort buffer to 2047 for DefaultSorter.
+    return Math.min(2047, availableMemoryMB);
+  }
+
   @Override
   public void write(Object key, Object value)
       throws IOException {

http://git-wip-us.apache.org/repos/asf/tez/blob/936ff8de/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
index c9c215d..408ec3b 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
@@ -124,6 +124,26 @@ public class TestDefaultSorter {
     }
   }
 
+  @Test(timeout = 5000)
+  public void testSortMBLimits() throws Exception {
+
+    assertTrue("Expected 2047", DefaultSorter.computeSortBufferSize(4096) == 2047);
+    assertTrue("Expected 2047", DefaultSorter.computeSortBufferSize(2047) == 2047);
+    assertTrue("Expected 1024", DefaultSorter.computeSortBufferSize(1024) == 1024);
+
+    try {
+      DefaultSorter.computeSortBufferSize(0);
+      fail("Should have thrown error for setting buffer size to 0");
+    } catch(RuntimeException re) {
+    }
+
+    try {
+      DefaultSorter.computeSortBufferSize(-100);
+      fail("Should have thrown error for setting buffer size to negative value");
+    } catch(RuntimeException re) {
+    }
+  }
+
   @Test(timeout = 30000)
   //Test TEZ-1977
   public void basicTest() throws IOException {


Mime
View raw message