tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gop...@apache.org
Subject git commit: TEZ-765. Allow tez.runtime.sort.threads > 1 to turn on PipelinedSorter (gopalv).
Date Mon, 27 Jan 2014 02:46:05 GMT
Updated Branches:
  refs/heads/master 43a9f608d -> c9859f72a


TEZ-765. Allow tez.runtime.sort.threads > 1 to turn on PipelinedSorter (gopalv).


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/c9859f72
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/c9859f72
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/c9859f72

Branch: refs/heads/master
Commit: c9859f72a9e6f527b31898f1da2cf8c9efbd31ed
Parents: 43a9f60
Author: Gopal V <gopalv@apache.org>
Authored: Sun Jan 26 18:44:50 2014 -0800
Committer: Gopal V <gopalv@apache.org>
Committed: Sun Jan 26 18:44:50 2014 -0800

----------------------------------------------------------------------
 .../tez/runtime/library/output/OnFileSortedOutput.java    | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/c9859f72/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
index aef1142..92e5d81 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OnFileSortedOutput.java
@@ -34,6 +34,7 @@ import org.apache.tez.runtime.api.events.DataMovementEvent;
 import org.apache.tez.runtime.api.events.VertexManagerEvent;
 import org.apache.tez.runtime.library.api.KeyValueWriter;
 import org.apache.tez.runtime.library.common.sort.impl.ExternalSorter;
+import org.apache.tez.runtime.library.common.sort.impl.PipelinedSorter;
 import org.apache.tez.runtime.library.common.sort.impl.dflt.DefaultSorter;
 import org.apache.tez.runtime.library.shuffle.common.ShuffleUtils;
 import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DataMovementEventPayloadProto;
@@ -60,12 +61,19 @@ public class OnFileSortedOutput implements LogicalOutput {
       throws IOException {
     this.startTime = System.nanoTime();
     this.outputContext = outputContext;
-    sorter = new DefaultSorter();
     this.conf = TezUtils.createConfFromUserPayload(outputContext.getUserPayload());
     // Initializing this parametr in this conf since it is used in multiple
     // places (wherever LocalDirAllocator is used) - TezTaskOutputFiles,
     // TezMerger, etc.
     this.conf.setStrings(TezJobConfig.LOCAL_DIRS, outputContext.getWorkDirs());
+    
+    if (this.conf.getInt(TezJobConfig.TEZ_RUNTIME_SORT_THREADS,
+        TezJobConfig.DEFAULT_TEZ_RUNTIME_SORT_THREADS) > 1) {
+      sorter = new PipelinedSorter();
+    } else {
+      sorter = new DefaultSorter();
+    }
+    
     sorter.initialize(outputContext, conf, numOutputs);
     return Collections.emptyList();
   }


Mime
View raw message