tez-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ss...@apache.org
Subject tez git commit: TEZ-1808. Job can fail since name of intermediate files can be too long in specific situation. Contributed by Tsuyoshi OZAWA. (cherry picked from commit 87b4c6b40cbe607b9d16b9d7e4a8f965757b5196)
Date Thu, 04 Dec 2014 22:00:37 GMT
Repository: tez
Updated Branches:
  refs/heads/branch-0.5.3 52318e26f -> 199ab0198


TEZ-1808. Job can fail since name of intermediate files can be too long
in specific situation. Contributed by Tsuyoshi OZAWA.
(cherry picked from commit 87b4c6b40cbe607b9d16b9d7e4a8f965757b5196)

Conflicts:
	CHANGES.txt
(cherry picked from commit 20c04e5534cddc20702fd025c407a2ccadc39fe9)


Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/199ab019
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/199ab019
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/199ab019

Branch: refs/heads/branch-0.5.3
Commit: 199ab0198d86344560675d2f46970de0f4053123
Parents: 52318e2
Author: Siddharth Seth <sseth@apache.org>
Authored: Wed Dec 3 17:12:20 2014 -0800
Committer: Siddharth Seth <sseth@apache.org>
Committed: Thu Dec 4 14:00:30 2014 -0800

----------------------------------------------------------------------
 CHANGES.txt                                               |  1 +
 .../common/shuffle/orderedgrouped/MergeManager.java       | 10 ++++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tez/blob/199ab019/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 945c645..5459112 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -18,6 +18,7 @@ ALL CHANGES:
   TEZ-1796. Use of DeprecationDelta broke build against 2.2 Hadoop.
   TEZ-1818. Problem loading tez-api-version-info.properties in case current context classloader
     in not pointing to Tez jars.
+  TEZ-1808. Job can fail since name of intermediate files can be too long in specific situation.
 
 Release 0.5.2: 2014-11-07
 

http://git-wip-us.apache.org/repos/asf/tez/blob/199ab019/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
index 6081f91..fd8b1ea 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/orderedgrouped/MergeManager.java
@@ -26,6 +26,8 @@ import java.util.Set;
 import java.util.TreeSet;
 
 import com.google.common.annotations.VisibleForTesting;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.commons.io.FilenameUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -130,6 +132,7 @@ public class MergeManager {
   private final int ifileReadAheadLength;
   private final int ifileBufferSize;
 
+  private AtomicInteger mergeFileSequenceId = new AtomicInteger(0);
 
   /**
    * Construct the MergeManager. Must call start before it becomes usable.
@@ -696,10 +699,13 @@ public class MergeManager {
       } else {
         namePart = file0.getPath().getName().toString();
       }
+
+      // namePart includes the suffix of the file. We need to remove it.
+      namePart = FilenameUtils.removeExtension(namePart);
       Path outputPath = localDirAllocator.getLocalPathForWrite(namePart, approxOutputSize,
conf);
-      outputPath = outputPath.suffix(Constants.MERGED_OUTPUT_PREFIX);
+      outputPath = outputPath.suffix(Constants.MERGED_OUTPUT_PREFIX + mergeFileSequenceId.getAndIncrement());
 
-      Writer writer = 
+      Writer writer =
         new Writer(conf, rfs, outputPath, 
                         (Class)ConfigUtils.getIntermediateInputKeyClass(conf), 
                         (Class)ConfigUtils.getIntermediateInputValueClass(conf),


Mime
View raw message