hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jiten...@apache.org
Subject [35/37] hadoop git commit: HDFS-8150. Make getFileChecksum fail for blocks under construction (Contributed by J.Andreina)
Date Thu, 14 May 2015 20:12:36 GMT
HDFS-8150. Make getFileChecksum fail for blocks under construction (Contributed by J.Andreina)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/def9136e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/def9136e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/def9136e

Branch: refs/heads/HDFS-7240
Commit: def9136e0259e118e6fd7b656260765d28ac9ae6
Parents: ffbb574
Author: Vinayakumar B <vinayakumarb@apache.org>
Authored: Thu May 14 15:54:51 2015 +0530
Committer: Vinayakumar B <vinayakumarb@apache.org>
Committed: Thu May 14 15:54:51 2015 +0530

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt      |  3 +++
 .../java/org/apache/hadoop/hdfs/DFSClient.java   |  8 ++++++++
 .../apache/hadoop/hdfs/TestGetFileChecksum.java  | 19 +++++++++++++++++++
 .../snapshot/TestSnapshotFileLength.java         | 17 +++++++++++++----
 4 files changed, 43 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/def9136e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0e6508b..4df18ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -758,6 +758,9 @@ Release 2.8.0 - UNRELEASED
     HDFS-7728. Avoid updating quota usage while loading edits.
     (Jing Zhao via wheat9)
 
+    HDFS-8150. Make getFileChecksum fail for blocks under construction
+    (J.Andreina via vinayakumarb)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/def9136e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 191ebc9..7908451 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1872,6 +1872,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
     if (null == blockLocations) {
       throw new FileNotFoundException("File does not exist: " + src);
     }
+    if (blockLocations.isUnderConstruction()) {
+      throw new IOException("Fail to get checksum, since file " + src
+          + " is under construction.");
+    }
     List<LocatedBlock> locatedblocks = blockLocations.getLocatedBlocks();
     final DataOutputBuffer md5out = new DataOutputBuffer();
     int bytesPerCRC = -1;
@@ -1891,6 +1895,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
         if (null == blockLocations) {
           throw new FileNotFoundException("File does not exist: " + src);
         }
+        if (blockLocations.isUnderConstruction()) {
+          throw new IOException("Fail to get checksum, since file " + src
+              + " is under construction.");
+        }
         locatedblocks = blockLocations.getLocatedBlocks();
         refetchBlocks = false;
       }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/def9136e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
index 0e56ba7..814261f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestGetFileChecksum.java
@@ -17,7 +17,12 @@
  */
 package org.apache.hadoop.hdfs;
 
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.Path;
 import org.junit.After;
@@ -68,6 +73,20 @@ public class TestGetFileChecksum {
   }
 
   @Test
+  public void testGetFileChecksumForBlocksUnderConstruction() {
+    try {
+      FSDataOutputStream file = dfs.create(new Path("/testFile"));
+      file.write("Performance Testing".getBytes());
+      dfs.getFileChecksum(new Path("/testFile"));
+      fail("getFileChecksum should fail for files "
+          + "with blocks under construction");
+    } catch (IOException ie) {
+      Assert.assertTrue(ie.getMessage().contains(
+          "Fail to get checksum, since file /testFile "
+              + "is under construction."));
+    }
+  }
+  @Test
   public void testGetFileChecksum() throws Exception {
     testGetFileChecksum(new Path("/foo"), BLOCKSIZE / 4);
     testGetFileChecksum(new Path("/bar"), BLOCKSIZE / 4 - 1);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/def9136e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index d53140f..90e499d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hdfs.server.namenode.snapshot;
 
 import java.io.ByteArrayOutputStream;
+import java.io.IOException;
 import java.io.PrintStream;
 
 import org.apache.hadoop.fs.FSDataOutputStream;
@@ -32,6 +33,8 @@ import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.CoreMatchers.not;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -111,8 +114,16 @@ public class TestSnapshotFileLength {
     // Append to the file.
     FSDataOutputStream out = hdfs.append(file1);
     // Nothing has been appended yet. All checksums should still be equal.
-    assertThat("file and snapshot checksums (open for append) are not equal",
-        hdfs.getFileChecksum(file1), is(snapChksum1));
+    // HDFS-8150:Fetching checksum for file under construction should fail
+    try {
+      hdfs.getFileChecksum(file1);
+      fail("getFileChecksum should fail for files "
+          + "with blocks under construction");
+    } catch (IOException ie) {
+      assertTrue(ie.getMessage().contains(
+          "Fail to get checksum, since file " + file1
+              + " is under construction."));
+    }
     assertThat("snapshot checksum (post-open for append) has changed",
         hdfs.getFileChecksum(file1snap1), is(snapChksum1));
     try {
@@ -122,8 +133,6 @@ public class TestSnapshotFileLength {
       assertThat("Wrong data size in snapshot.",
           dataFromSnapshot.length, is(origLen));
       // Verify that checksum didn't change
-      assertThat("snapshot file checksum (pre-close) has changed",
-          hdfs.getFileChecksum(file1), is(snapChksum1));
       assertThat("snapshot checksum (post-append) has changed",
           hdfs.getFileChecksum(file1snap1), is(snapChksum1));
     } finally {


Mime
View raw message