hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject hadoop git commit: HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files. Contributed by Rakesh R.
Date Mon, 02 Nov 2015 15:48:49 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 2529464f0 -> 90e140551


HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files. Contributed by Rakesh
R.

Change-Id: Ia3ae582405e741ca8e90d9255ab9b95d085e5fa8


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90e14055
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90e14055
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90e14055

Branch: refs/heads/trunk
Commit: 90e14055168afdb93fa8089158c03a6a694e066c
Parents: 2529464
Author: Zhe Zhang <zhz@apache.org>
Authored: Mon Nov 2 07:48:30 2015 -0800
Committer: Zhe Zhang <zhz@apache.org>
Committed: Mon Nov 2 07:48:30 2015 -0800

----------------------------------------------------------------------
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt     |   3 +
 .../TestErasureCodingPolicyWithSnapshot.java    | 199 +++++++++++++++++++
 2 files changed, 202 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e14055/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0bbc60d..3b2d997 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -189,6 +189,9 @@ Trunk (Unreleased)
 
     HDFS-9323. Randomize the DFSStripedOutputStreamWithFailure tests. (szetszwo)
 
+    HDFS-8777. Erasure Coding: add tests for taking snapshots on EC files. 
+    (Rakesh R via zhz)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/90e14055/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
new file mode 100644
index 0000000..515763c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FsShell;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestErasureCodingPolicyWithSnapshot {
+  private MiniDFSCluster cluster;
+  private DistributedFileSystem fs;
+  private Configuration conf;
+
+  private final static short GROUP_SIZE = StripedFileTestUtil.NUM_DATA_BLOCKS
+      + StripedFileTestUtil.NUM_PARITY_BLOCKS;
+  private final static int SUCCESS = 0;
+  private final ErasureCodingPolicy sysDefaultPolicy = ErasureCodingPolicyManager
+      .getSystemDefaultPolicy();
+
+  @Before
+  public void setupCluster() throws IOException {
+    conf = new HdfsConfiguration();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
+    cluster.waitActive();
+    fs = cluster.getFileSystem();
+  }
+
+  @After
+  public void shutdownCluster() throws IOException {
+    if (cluster != null) {
+      cluster.shutdown();
+    }
+  }
+
+  /**
+   * Test correctness of successive snapshot creation and deletion with erasure
+   * coding policies. Create snapshot of ecDir's parent directory.
+   */
+  @Test(timeout = 120000)
+  public void testSnapshotsOnErasureCodingDirsParentDir() throws Exception {
+    final int len = 1024;
+    final Path ecDirParent = new Path("/parent");
+    final Path ecDir = new Path(ecDirParent, "ecdir");
+    final Path ecFile = new Path(ecDir, "ecfile");
+    fs.mkdirs(ecDir);
+    fs.allowSnapshot(ecDirParent);
+    // set erasure coding policy
+    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+    DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
+    String contents = DFSTestUtil.readFile(fs, ecFile);
+    final Path snap1 = fs.createSnapshot(ecDirParent, "snap1");
+    final Path snap1ECDir = new Path(snap1, ecDir.getName());
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        fs.getErasureCodingPolicy(snap1ECDir));
+
+    // Now delete the dir which has erasure coding policy. Re-create the dir again, and
+    // take another snapshot
+    fs.delete(ecDir, true);
+    fs.mkdir(ecDir, FsPermission.getDirDefault());
+    final Path snap2 = fs.createSnapshot(ecDirParent, "snap2");
+    final Path snap2ECDir = new Path(snap2, ecDir.getName());
+    assertNull("Expected null erasure coding policy",
+        fs.getErasureCodingPolicy(snap2ECDir));
+
+    // Make dir again with system default ec policy
+    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+    final Path snap3 = fs.createSnapshot(ecDirParent, "snap3");
+    final Path snap3ECDir = new Path(snap3, ecDir.getName());
+    // Check that snap3's ECPolicy has the correct settings
+    ErasureCodingPolicy ezSnap3 = fs.getErasureCodingPolicy(snap3ECDir);
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        ezSnap3);
+
+    // Check that older snapshots still have the old ECPolicy settings
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        fs.getErasureCodingPolicy(snap1ECDir));
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        fs.getErasureCodingPolicy(snap2ECDir));
+
+    // Verify contents of the snapshotted file
+    final Path snapshottedECFile = new Path(snap1.toString() + "/"
+        + ecDir.getName() + "/" + ecFile.getName());
+    assertEquals("Contents of snapshotted file have changed unexpectedly",
+        contents, DFSTestUtil.readFile(fs, snapshottedECFile));
+
+    // Now delete the snapshots out of order and verify the EC policy
+    // correctness
+    fs.deleteSnapshot(ecDirParent, snap2.getName());
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        fs.getErasureCodingPolicy(snap1ECDir));
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        fs.getErasureCodingPolicy(snap3ECDir));
+    fs.deleteSnapshot(ecDirParent, snap1.getName());
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        fs.getErasureCodingPolicy(snap3ECDir));
+  }
+
+  /**
+   * Test creation of snapshot on directory has erasure coding policy.
+   */
+  @Test(timeout = 120000)
+  public void testSnapshotsOnErasureCodingDir() throws Exception {
+    final Path ecDir = new Path("/ecdir");
+    fs.mkdirs(ecDir);
+    fs.allowSnapshot(ecDir);
+
+    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+    final Path snap1 = fs.createSnapshot(ecDir, "snap1");
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        fs.getErasureCodingPolicy(snap1));
+  }
+
+  /**
+   * Test verify erasure coding policy is present after restarting the NameNode.
+   */
+  @Test(timeout = 120000)
+  public void testSnapshotsOnErasureCodingDirAfterNNRestart() throws Exception {
+    final Path ecDir = new Path("/ecdir");
+    fs.mkdirs(ecDir);
+    fs.allowSnapshot(ecDir);
+
+    // set erasure coding policy
+    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+    final Path snap1 = fs.createSnapshot(ecDir, "snap1");
+    ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1);
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        ecSnap);
+
+    // save namespace, restart namenode, and check ec policy correctness.
+    fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+    fs.saveNamespace();
+    fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+    cluster.restartNameNode(true);
+
+    ErasureCodingPolicy ecSnap1 = fs.getErasureCodingPolicy(snap1);
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        ecSnap1);
+    assertEquals("Got unexpected ecSchema", ecSnap.getSchema(),
+        ecSnap1.getSchema());
+  }
+
+  /**
+   * Test copy a snapshot will not preserve its erasure coding policy info.
+   */
+  @Test(timeout = 120000)
+  public void testCopySnapshotWillNotPreserveErasureCodingPolicy()
+      throws Exception {
+    final int len = 1024;
+    final Path ecDir = new Path("/ecdir");
+    final Path ecFile = new Path(ecDir, "ecFile");
+    fs.mkdirs(ecDir);
+    fs.allowSnapshot(ecDir);
+
+    // set erasure coding policy
+    fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy);
+    DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED);
+    final Path snap1 = fs.createSnapshot(ecDir, "snap1");
+
+    Path snap1Copy = new Path(ecDir.toString() + "-copy");
+    final Path snap1CopyECDir = new Path("/ecdir-copy");
+    String[] argv = new String[] { "-cp", "-px", snap1.toUri().toString(),
+        snap1Copy.toUri().toString() };
+    int ret = ToolRunner.run(new FsShell(conf), argv);
+    assertEquals("cp -px is not working on a snapshot", SUCCESS, ret);
+
+    assertNull("Got unexpected erasure coding policy",
+        fs.getErasureCodingPolicy(snap1CopyECDir));
+    assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy,
+        fs.getErasureCodingPolicy(snap1));
+  }
+}


Mime
View raw message