tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rbalamo...@apache.org
Subject tez git commit: TEZ-2505. PipelinedSorter uses Comparator objects concurrently from multiple threads (rbalamohan)
Date Tue, 02 Jun 2015 08:19:10 GMT
Repository: tez
Updated Branches:
  refs/heads/master 4c48283ee -> 237d1895d


TEZ-2505. PipelinedSorter uses Comparator objects concurrently from multiple threads (rbalamohan)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/237d1895
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/237d1895
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/237d1895

Branch: refs/heads/master
Commit: 237d1895d224fc07466e0c4e494960b32d7a69aa
Parents: 4c48283
Author: Rajesh Balamohan <rbalamohan@apache.org>
Authored: Tue Jun 2 13:50:16 2015 +0530
Committer: Rajesh Balamohan <rbalamohan@apache.org>
Committed: Tue Jun 2 13:50:16 2015 +0530

----------------------------------------------------------------------
 CHANGES.txt                                               |  1 +
 .../runtime/library/common/sort/impl/PipelinedSorter.java | 10 ++++------
 2 files changed, 5 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/237d1895/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 116a387..60e2b89 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -31,6 +31,7 @@ Release 0.7.1: Unreleased
 INCOMPATIBLE CHANGES
 
 ALL CHANGES:
+  TEZ-2505. PipelinedSorter uses Comparator objects concurrently from multiple threads.
   TEZ-2504. Tez UI: tables - show status column without scrolling, numeric 0 shown as Not
available
   TEZ-2478. Move OneToOne routing to store events in Tasks.
   TEZ-2482. Tez UI: Mouse events not working on IE11

http://git-wip-us.apache.org/repos/asf/tez/blob/237d1895/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
index 6e4d72e..e3824be 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/PipelinedSorter.java
@@ -107,8 +107,6 @@ public class PipelinedSorter extends ExternalSorter {
 
   private final ArrayList<TezSpillRecord> indexCacheList =
     new ArrayList<TezSpillRecord>();
-  private int totalIndexCacheMemory;
-  private int indexCacheMemoryLimit;
 
   private final boolean pipelinedShuffle;
 
@@ -133,8 +131,6 @@ public class PipelinedSorter extends ExternalSorter {
 
     //sanity checks
     final long sortmb = this.availableMemoryMb;
-    indexCacheMemoryLimit = this.conf.getInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES,
-                                       TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES_DEFAULT);
 
     // buffers and accounting
     long maxMemUsage = sortmb << 20;
@@ -244,7 +240,8 @@ public class PipelinedSorter extends ExternalSorter {
         }
       }
       Preconditions.checkArgument(listIterator.hasNext(), "block iterator should not be empty");
-      span = new SortSpan((ByteBuffer)listIterator.next().clear(), (1024*1024), perItem,
this.comparator);
+      span = new SortSpan((ByteBuffer)listIterator.next().clear(), (1024*1024),
+          perItem, ConfigUtils.getIntermediateOutputKeyComparator(this.conf));
     } else {
       // queue up the sort
       SortTask task = new SortTask(span, sorter);
@@ -755,7 +752,8 @@ public class PipelinedSorter extends ExternalSorter {
           items = 1024*1024;
           perItem = 16;
         }
-        newSpan = new SortSpan(remaining, items, perItem, this.comparator);
+        newSpan = new SortSpan(remaining, items, perItem,
+            ConfigUtils.getIntermediateOutputKeyComparator(conf));
         newSpan.index = index+1;
         LOG.info(String.format("New Span%d.length = %d, perItem = %d", newSpan.index, newSpan
             .length(), perItem) + ", counter:" + mapOutputRecordCounter.getValue());


Mime
View raw message