tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject tez git commit: TEZ-2505. PipelinedSorter uses Comparator objects concurrently from multiple threads (rbalamohan)
Date Tue, 02 Jun 2015 08:21:35 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.7 9e0f2e5df -> feca51a3b


TEZ-2505. PipelinedSorter uses Comparator objects concurrently from multiple threads (rbalamohan)

(cherry picked from commit 237d1895d224fc07466e0c4e494960b32d7a69aa)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/feca51a3
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/feca51a3
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/feca51a3

Branch: refs/heads/branch-0.7
Commit: feca51a3b22bcf6cf8f8f18de2c17d44a39e6f17
Parents: 9e0f2e5
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Tue Jun 2 13:50:16 2015 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Tue Jun 2 13:51:33 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                               |  1 +
 .../runtime/library/common/sort/impl/PipelinedSorter.java | 10 ++++------
 2 files changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/feca51a3/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 58ab704..eca3e26 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -7,6 +7,7 @@ Release 0.7.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2505. PipelinedSorter uses Comparator objects concurrently from multiple threads.
   TEZ-2504. Tez UI: tables - show status column without scrolling, numeric 0 shown as Not
available
   TEZ-2478. Move OneToOne routing to store events in Tasks.
   TEZ-2482. Tez UI: Mouse events not working on IE11

http://git-wip-us.apache.org/repos/asf/tez/blob/feca51a3/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
index 661f54c..e7cd81c 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
@@ -106,8 +106,6 @@ public class PipelinedSorter extends ExternalSorter {
 
   private final ArrayList<TezSpillRecord> indexCacheList =
     new ArrayList<TezSpillRecord>();
-  private int totalIndexCacheMemory;
-  private int indexCacheMemoryLimit;
 
   private final boolean pipelinedShuffle;
   private final boolean finalMergeEnabled;
@@ -143,8 +141,6 @@ public class PipelinedSorter extends ExternalSorter {
 
     //sanity checks
     final long sortmb = this.availableMemoryMb;
-    indexCacheMemoryLimit = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES,
-                                       TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT);
 
     // buffers and accounting
     long maxMemUsage = sortmb << 20;
@@ -254,7 +250,8 @@ public class PipelinedSorter extends ExternalSorter {
         }
       }
       Preconditions.checkArgument(listIterator.hasNext(), "block iterator should not be empty");
-      span = new SortSpan((ByteBuffer)listIterator.next().clear(), (1024*1024), perItem,
this.comparator);
+      span = new SortSpan((ByteBuffer)listIterator.next().clear(), (1024*1024),
+          perItem, ConfigUtils.getIntermediateOutputKeyComparator(this.conf));
     } else {
       // queue up the sort
       SortTask task = new SortTask(span, sorter);
@@ -695,7 +692,8 @@ public class PipelinedSorter extends ExternalSorter {
           items = 1024*1024;
           perItem = 16;
         }
-        newSpan = new SortSpan(remaining, items, perItem, this.comparator);
+        newSpan = new SortSpan(remaining, items, perItem,
+            ConfigUtils.getIntermediateOutputKeyComparator(conf));
         newSpan.index = index+1;
         LOG.info(String.format("New Span%d.length = %d, perItem = %d", newSpan.index, newSpan
             .length(), perItem) + ", counter:" + mapOutputRecordCounter.getValue());


Mime
View raw message