hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From s..@apache.org
Subject [02/50] [abbrv] hadoop git commit: HDFS-13936. Multipart upload to HDFS to support 0 byte upload. Contributed by Ewan Higgs.
Date Fri, 05 Oct 2018 22:06:31 GMT
HDFS-13936. Multipart upload to HDFS to support 0 byte upload. Contributed by Ewan Higgs.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fab6886
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fab6886
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fab6886

Branch: refs/heads/HDFS-12943
Commit: 6fab6886f652492573734b832ca0375459a82775
Parents: a383ac4
Author: Ewan Higgs <ehiggs@apache.org>
Authored: Tue Oct 2 14:03:28 2018 +0200
Committer: Ewan Higgs <ehiggs@apache.org>
Committed: Tue Oct 2 14:05:35 2018 +0200

----------------------------------------------------------------------
 .../hadoop/fs/FileSystemMultipartUploader.java  | 25 +++++++++++++++-----
 .../AbstractContractMultipartUploaderTest.java  | 22 +++++++++++++++++
 2 files changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fab6886/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
index 690194d..94c7861 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystemMultipartUploader.java
@@ -100,6 +100,14 @@ public class FileSystemMultipartUploader extends MultipartUploader {
     return fs.getPathHandle(status);
   }
 
+  private long totalPartsLen(List<Path> partHandles) throws IOException {
+    long totalLen = 0;
+    for (Path p: partHandles) {
+      totalLen += fs.getFileStatus(p).getLen();
+    }
+    return totalLen;
+  }
+
   @Override
   @SuppressWarnings("deprecation") // rename w/ OVERWRITE
   public PathHandle complete(Path filePath,
@@ -127,12 +135,17 @@ public class FileSystemMultipartUploader extends MultipartUploader {
         .collect(Collectors.toList());
 
     Path collectorPath = createCollectorPath(filePath);
-    Path filePathInsideCollector = mergePaths(collectorPath,
-        new Path(Path.SEPARATOR + filePath.getName()));
-    fs.create(filePathInsideCollector).close();
-    fs.concat(filePathInsideCollector,
-        partHandles.toArray(new Path[handles.size()]));
-    fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE);
+    boolean emptyFile = totalPartsLen(partHandles) == 0;
+    if (emptyFile) {
+      fs.create(filePath).close();
+    } else {
+      Path filePathInsideCollector = mergePaths(collectorPath,
+          new Path(Path.SEPARATOR + filePath.getName()));
+      fs.create(filePathInsideCollector).close();
+      fs.concat(filePathInsideCollector,
+          partHandles.toArray(new Path[handles.size()]));
+      fs.rename(filePathInsideCollector, filePath, Options.Rename.OVERWRITE);
+    }
     fs.delete(collectorPath, true);
     return getPathHandle(filePath);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fab6886/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
index 85a6861..7cee5a6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/contract/AbstractContractMultipartUploaderTest.java
@@ -165,6 +165,28 @@ public abstract class AbstractContractMultipartUploaderTest extends
   }
 
   /**
+   * Assert that a multipart upload is successful when a single empty part is
+   * uploaded.
+   * @throws Exception failure
+   */
+  @Test
+  public void testMultipartUploadEmptyPart() throws Exception {
+    FileSystem fs = getFileSystem();
+    Path file = path("testMultipartUpload");
+    MultipartUploader mpu = MultipartUploaderFactory.get(fs, null);
+    UploadHandle uploadHandle = mpu.initialize(file);
+    List<Pair<Integer, PartHandle>> partHandles = new ArrayList<>();
+    MessageDigest origDigest = DigestUtils.getMd5Digest();
+    byte[] payload = new byte[0];
+    origDigest.update(payload);
+    InputStream is = new ByteArrayInputStream(payload);
+    PartHandle partHandle = mpu.putPart(file, is, 0, uploadHandle,
+        payload.length);
+      partHandles.add(Pair.of(0, partHandle));
+    completeUpload(file, mpu, uploadHandle, partHandles, origDigest, 0);
+  }
+
+  /**
    * Assert that a multipart upload is successful even when the parts are
    * given in the reverse order.
    */


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message