tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject tez git commit: TEZ-2538. ADDITIONAL_SPILL_COUNT wrongly populated for DefaultSorter with multiple partitions (rbalamohan)
Date Wed, 16 Dec 2015 22:44:52 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 d2c3e5d21 -> 88cbd2edb


TEZ-2538. ADDITIONAL_SPILL_COUNT wrongly populated for DefaultSorter with multiple partitions
(rbalamohan)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/88cbd2ed
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/88cbd2ed
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/88cbd2ed

Branch: refs/heads/branch-0.7
Commit: 88cbd2edb6c8259f5dddd63eddc2505013f59bdf
Parents: d2c3e5d
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Thu Dec 17 04:14:18 2015 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Thu Dec 17 04:14:18 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                             |  1 +
 .../library/common/sort/impl/dflt/DefaultSorter.java    |  7 ++++++-
 .../library/common/sort/impl/TestPipelinedSorter.java   | 12 ++++++++++++
 .../common/sort/impl/dflt/TestDefaultSorter.java        |  4 ++--
 4 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/88cbd2ed/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 71ab3ea..7eefa07 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ INCOMPATIBLE CHANGES
   TEZ-2949. Allow duplicate dag names within session for Tez.
 
 ALL CHANGES
+  TEZ-2538. ADDITIONAL_SPILL_COUNT wrongly populated for DefaultSorter with multiple partitions.
   TEZ-3006. Remove unused import in TestHistoryParser.
   TEZ-2918. Make progress notifications in IOs
   TEZ-2979. FlakyTest: org.apache.tez.history.TestHistoryParser.

http://git-wip-us.apache.org/repos/asf/tez/blob/88cbd2ed/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
index 6fe98b5..7cb7742 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/dflt/DefaultSorter.java
@@ -805,7 +805,6 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab
     } else {
       if (numSpills > 0) {
         additionalSpillBytesWritten.increment(compLength);
-        numAdditionalSpills.increment(1);
         // Reset the value will be set during the final merge.
         outputBytesWithOverheadCounter.setValue(0);
       } else {
@@ -910,6 +909,9 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab
       ++numSpills;
       if (!isFinalMergeEnabled()) {
         numShuffleChunks.setValue(numSpills);
+      } else if (numSpills > 1) {
+        //Increment only when there was atleast one previous spill
+        numAdditionalSpills.increment(1);
       }
     } finally {
       if (out != null) out.close();
@@ -982,6 +984,9 @@ public final class DefaultSorter extends ExternalSorter implements IndexedSortab
       ++numSpills;
       if (!isFinalMergeEnabled()) {
         numShuffleChunks.setValue(numSpills);
+      } else if (numSpills > 1) {
+        //Increment only when there is atleast one previous spill
+        numAdditionalSpills.increment(1);
       }
     } finally {
       if (out != null) out.close();

http://git-wip-us.apache.org/repos/asf/tez/blob/88cbd2ed/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
index f5076a6..1b92347 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/TestPipelinedSorter.java
@@ -331,6 +331,18 @@ public class TestPipelinedSorter {
     sorter.close();
   }
 
+  @Test
+  public void testCountersWithMultiplePartitions() throws IOException {
+    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);
+    this.numOutputs = 5;
+    this.initialAvailableMem = 5 * 1024 * 1024;
+    PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
+        initialAvailableMem, 0);
+
+    writeData(sorter, 10000, 100);
+    verifyCounters(sorter, outputContext);
+  }
+
   public void basicTest(int partitions, int numKeys, int keySize,
       long initialAvailableMem, int blockSize) throws IOException {
     this.numOutputs = partitions; // single output

http://git-wip-us.apache.org/repos/asf/tez/blob/88cbd2ed/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
index 91112da..1da48c2 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/sort/impl/dflt/TestDefaultSorter.java
@@ -253,7 +253,7 @@ public class TestDefaultSorter {
     conf.setLong(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 1);
     context.requestInitialMemory(ExternalSorter.getInitialMemoryRequirement(conf,
             context.getTotalMemoryAvailableToTask()), handler);
-    DefaultSorter sorter = new DefaultSorter(context, conf, 1, handler.getMemoryAssigned());
+    DefaultSorter sorter = new DefaultSorter(context, conf, 5, handler.getMemoryAssigned());
 
     //Write 1000 keys each of size 1000, (> 1 spill should happen)
     try {
@@ -295,7 +295,7 @@ public class TestDefaultSorter {
     MemoryUpdateCallbackHandler handler = new MemoryUpdateCallbackHandler();
     context.requestInitialMemory(ExternalSorter.getInitialMemoryRequirement(conf,
         context.getTotalMemoryAvailableToTask()), handler);
-    DefaultSorter sorter = new DefaultSorter(context, conf, 1, handler.getMemoryAssigned());
+    DefaultSorter sorter = new DefaultSorter(context, conf, 5, handler.getMemoryAssigned());
 
     //no data written. Empty
     try {


Mime
View raw message