hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ste...@apache.org
Subject [1/2] hadoop git commit: HADOOP-15469. S3A directory committer commit job fails if _temporary directory created under dest. Contributed by Steve Loughran.
Date Thu, 17 May 2018 18:09:29 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.1 f729faaab -> 170f1040d
  refs/heads/trunk c0ec061e2 -> cc3600aab


HADOOP-15469. S3A directory committer commit job fails if _temporary directory created under
dest.
Contributed by Steve Loughran.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/170f1040
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/170f1040
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/170f1040

Branch: refs/heads/branch-3.1
Commit: 170f1040d46f9b1a084e6637def91e9864446acc
Parents: f729faa
Author: Steve Loughran <stevel@apache.org>
Authored: Thu May 17 19:08:05 2018 +0100
Committer: Steve Loughran <stevel@apache.org>
Committed: Thu May 17 19:08:05 2018 +0100

----------------------------------------------------------------------
 .../fs/s3a/commit/staging/DirectoryStagingCommitter.java     | 7 ++-----
 .../commit/staging/TestStagingDirectoryOutputCommitter.java  | 8 +++-----
 2 files changed, 5 insertions(+), 10 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/170f1040/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
index 03dfed2..3eda24f 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/commit/staging/DirectoryStagingCommitter.java
@@ -93,11 +93,8 @@ public class DirectoryStagingCommitter extends StagingCommitter {
     Configuration fsConf = fs.getConf();
     switch (getConflictResolutionMode(context, fsConf)) {
     case FAIL:
-      // this was checked in setupJob, but this avoids some cases where
-      // output was created while the job was processing
-      if (fs.exists(outputPath)) {
-        throw new PathExistsException(outputPath.toString(), E_DEST_EXISTS);
-      }
+      // this was checked in setupJob; temporary files may have been
+      // created, so do not check again.
       break;
     case APPEND:
       // do nothing

http://git-wip-us.apache.org/repos/asf/hadoop/blob/170f1040/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
index 4f0189e..f5b8d1f 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/commit/staging/TestStagingDirectoryOutputCommitter.java
@@ -67,15 +67,14 @@ public class TestStagingDirectoryOutputCommitter
     pathExists(mockS3, OUTPUT_PATH);
     final DirectoryStagingCommitter committer = newJobCommitter();
 
+    // this should fail
     intercept(PathExistsException.class,
         InternalCommitterConstants.E_DEST_EXISTS,
         "Should throw an exception because the path exists",
         () -> committer.setupJob(getJob()));
 
-    intercept(PathExistsException.class,
-        InternalCommitterConstants.E_DEST_EXISTS,
-        "Should throw an exception because the path exists",
-        () -> committer.commitJob(getJob()));
+    // but there are no checks in job commit (HADOOP-15469)
+    committer.commitJob(getJob());
 
     reset(mockS3);
     pathDoesNotExist(mockS3, OUTPUT_PATH);
@@ -87,7 +86,6 @@ public class TestStagingDirectoryOutputCommitter
     reset(mockS3);
     pathDoesNotExist(mockS3, OUTPUT_PATH);
     committer.commitJob(getJob());
-    verifyExistenceChecked(mockS3, OUTPUT_PATH);
     verifyCompletion(mockS3);
   }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message