tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject git commit: TEZ-981. Port MAPREDUCE-3685. (sseth)
Date Thu, 27 Mar 2014 17:41:59 GMT
Repository: incubator-tez
Updated Branches:
  refs/heads/master 57b61c22f -> cde19354d


TEZ-981. Port MAPREDUCE-3685. (sseth)


Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/cde19354
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/cde19354
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/cde19354

Branch: refs/heads/master
Commit: cde19354d5a7ee73a2fff5312d5d5ecb995da428
Parents: 57b61c2
Author: Siddharth Seth <sseth@apache.org>
Authored: Thu Mar 27 10:41:40 2014 -0700
Committer: Siddharth Seth <sseth@apache.org>
Committed: Thu Mar 27 10:41:40 2014 -0700

----------------------------------------------------------------------
 .../common/shuffle/impl/MergeManager.java       |  4 ++--
 .../library/common/sort/impl/TezMerger.java     | 22 --------------------
 2 files changed, 2 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cde19354/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
index 5a952ba..8a09aab 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/MergeManager.java
@@ -634,7 +634,7 @@ public class MergeManager {
   private class OnDiskMerger extends MergeThread<Path> {
     
     public OnDiskMerger(MergeManager manager) {
-      super(manager, Integer.MAX_VALUE, exceptionReporter);
+      super(manager, ioSortFactor, exceptionReporter);
       setName("DiskToDiskMerger [" + TezUtils.cleanVertexName(inputContext.getSourceVertexName())
+ "]");
       setDaemon(true);
     }
@@ -888,7 +888,7 @@ public class MergeManager {
       diskSegments.addAll(0, memDiskSegments);
       memDiskSegments.clear();
       TezRawKeyValueIterator diskMerge = TezMerger.merge(
-          job, fs, keyClass, valueClass, diskSegments,
+          job, fs, keyClass, valueClass, codec, diskSegments,
           ioSortFactor, numInMemSegments, tmpDir, comparator,
           nullProgressable, false, spilledRecordsCounter, null, additionalBytesRead, null);
       diskSegments.clear();

http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/cde19354/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
index ae03370..ee62b70 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/sort/impl/TezMerger.java
@@ -171,28 +171,6 @@ public class TezMerger {
   }
 
   public static <K extends Object, V extends Object>
-    TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
-                            Class keyClass, Class valueClass,
-                            List<Segment> segments,
-                            int mergeFactor, int inMemSegments, Path tmpDir,
-                            RawComparator comparator, Progressable reporter,
-                            boolean sortSegments,
-                            TezCounter readsCounter,
-                            TezCounter writesCounter,
-                            TezCounter bytesReadCounter,
-                            Progress mergePhase)
-      throws IOException {
-    return new MergeQueue(conf, fs, segments, comparator, reporter,
-                           sortSegments, false).merge(keyClass, valueClass,
-                                               mergeFactor, inMemSegments,
-                                               tmpDir,
-                                               readsCounter, writesCounter,
-                                               bytesReadCounter,
-                                               mergePhase);
-  }
-
-
-  static <K extends Object, V extends Object>
   TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                           Class keyClass, Class valueClass,
                           CompressionCodec codec,


Mime
View raw message