tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject tez git commit: TEZ-1808. Job can fail since name of intermediate files can be too long in specific situation. Contributed by Tsuyoshi OZAWA.
Date Thu, 04 Dec 2014 01:13:00 GMT
Repository: tez
Updated Branches:
  refs/heads/master ac26ade40 -> 87b4c6b40


TEZ-1808. Job can fail since name of intermediate files can be too long
in specific situation. Contributed by Tsuyoshi OZAWA.


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/87b4c6b4
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/87b4c6b4
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/87b4c6b4

Branch: refs/heads/master
Commit: 87b4c6b40cbe607b9d16b9d7e4a8f965757b5196
Parents: ac26ade
Author: Siddharth Seth <sseth@apache.org>
Authored: Wed Dec 3 17:12:20 2014 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Wed Dec 3 17:12:20 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                               |  1 +
 .../common/shuffle/orderedgrouped/MergeManager.java       | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/87b4c6b4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4ad652b..1348ab0 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -28,6 +28,7 @@ ALL CHANGES:
   TEZ-1697. DAG submission fails if a local resource added is already part of tez.lib.uris
   TEZ-1800. Integer overflow in ExternalSorter.getInitialMemoryRequirement()
   TEZ-1790. DeallocationTaskRequest may been handled before corresponding AllocationTaskRequest
in local mode
+  TEZ-1808. Job can fail since name of intermediate files can be too long in specific situation.
 
 Release 0.5.3: Unreleased
 

http://git-wip-us.apache.org/repos/asf/tez/blob/87b4c6b4/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
index 6081f91..fd8b1ea 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
@@ -26,6 +26,8 @@ import java.util.Set;
 import java.util.TreeSet;
 
 import com.google.common.annotations.VisibleForTesting;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -130,6 +132,7 @@ public class MergeManager {
   private final int ifileReadAheadLength;
   private final int ifileBufferSize;
 
+  private AtomicInteger mergeFileSequenceId = new AtomicInteger(0);
 
   /**
    * Construct the MergeManager. Must call start before it becomes usable.
@@ -696,10 +699,13 @@ public class MergeManager {
       } else {
         namePart = file0.getPath().getName().toString();
       }
+
+      // namePart includes the suffix of the file. We need to remove it.
+      namePart = FilenameUtils.removeExtension(namePart);
       Path outputPath = localDirAllocator.getLocalPathForWrite(namePart, approxOutputSize,
conf);
-      outputPath = outputPath.suffix(Constants.MERGED_OUTPUT_PREFIX);
+      outputPath = outputPath.suffix(Constants.MERGED_OUTPUT_PREFIX + mergeFileSequenceId.getAndIncrement());
 
-      Writer writer = 
+      Writer writer =
         new Writer(conf, rfs, outputPath, 
                         (Class)ConfigUtils.getIntermediateInputKeyClass(conf), 
                         (Class)ConfigUtils.getIntermediateInputValueClass(conf),


Mime
View raw message