hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [2/2] hadoop git commit: HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file includes non-snapshotted content. Contributed by Charles Lamb. (cherry picked from commit f2d7a67a2c1d9dde10ed3171fdec65dff885afcc)
Date Wed, 04 Mar 2015 02:11:24 GMT
HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file includes non-snapshotted
content. Contributed by Charles Lamb.
(cherry picked from commit f2d7a67a2c1d9dde10ed3171fdec65dff885afcc)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bce3d442
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bce3d442
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bce3d442

Branch: refs/heads/branch-2
Commit: bce3d442ff08ee1e730b0bac112439d6a6931917
Parents: 65bfde5
Author: Aaron T. Myers <atm@apache.org>
Authored: Tue Mar 3 18:08:59 2015 -0800
Committer: Aaron T. Myers <atm@apache.org>
Committed: Tue Mar 3 18:09:31 2015 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |  3 +++
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  3 +++
 .../snapshot/TestSnapshotFileLength.java        | 25 +++++++++++++++++---
 3 files changed, 28 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bce3d442/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 69a410f..bff45bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -774,6 +774,9 @@ Release 2.7.0 - UNRELEASED
     HDFS-6565. Use jackson instead jetty json in hdfs-client.
     (Akira Ajisaka via wheat9)
 
+    HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file
+    includes non-snapshotted content. (Charles Lamb via atm)
+
     BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
 
       HDFS-7720. Quota by Storage Type API, tools and ClientNameNode

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bce3d442/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 6f96126..ba6a1d1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2219,6 +2219,9 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
 
     // get block checksum for each block
     long remaining = length;
+    if (src.contains(HdfsConstants.SEPARATOR_DOT_SNAPSHOT_DIR_SEPARATOR)) {
+      remaining = Math.min(length, blockLocations.getFileLength());
+    }
     for(int i = 0; i < locatedblocks.size() && remaining > 0; i++) {
       if (refetchBlocks) {  // refetch to get fresh tokens
         blockLocations = callGetBlockLocations(namenode, src, 0, length);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bce3d442/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
index 98aafc1..d53140f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotFileLength.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.hdfs.server.namenode.snapshot;
 import java.io.ByteArrayOutputStream;
 import java.io.PrintStream;
 
-
 import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileChecksum;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.hdfs.AppendTestUtil;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -29,8 +29,9 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.*;
-
+import static org.hamcrest.CoreMatchers.not;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataInputStream;
@@ -103,17 +104,35 @@ public class TestSnapshotFileLength {
     Path file1snap1
         = SnapshotTestHelper.getSnapshotPath(sub, snapshot1, file1Name);
 
+    final FileChecksum snapChksum1 = hdfs.getFileChecksum(file1snap1);
+    assertThat("file and snapshot file checksums are not equal",
+        hdfs.getFileChecksum(file1), is(snapChksum1));
+
     // Append to the file.
     FSDataOutputStream out = hdfs.append(file1);
+    // Nothing has been appended yet. All checksums should still be equal.
+    assertThat("file and snapshot checksums (open for append) are not equal",
+        hdfs.getFileChecksum(file1), is(snapChksum1));
+    assertThat("snapshot checksum (post-open for append) has changed",
+        hdfs.getFileChecksum(file1snap1), is(snapChksum1));
     try {
       AppendTestUtil.write(out, 0, toAppend);
       // Test reading from snapshot of file that is open for append
       byte[] dataFromSnapshot = DFSTestUtil.readFileBuffer(hdfs, file1snap1);
       assertThat("Wrong data size in snapshot.",
           dataFromSnapshot.length, is(origLen));
+      // Verify that checksum didn't change
+      assertThat("snapshot file checksum (pre-close) has changed",
+          hdfs.getFileChecksum(file1), is(snapChksum1));
+      assertThat("snapshot checksum (post-append) has changed",
+          hdfs.getFileChecksum(file1snap1), is(snapChksum1));
     } finally {
       out.close();
     }
+    assertThat("file and snapshot file checksums (post-close) are equal",
+        hdfs.getFileChecksum(file1), not(snapChksum1));
+    assertThat("snapshot file checksum (post-close) has changed",
+        hdfs.getFileChecksum(file1snap1), is(snapChksum1));
 
     // Make sure we can read the entire file via its non-snapshot path.
     fileStatus = hdfs.getFileStatus(file1);


Mime
View raw message