hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From x...@apache.org
Subject hadoop git commit: HDFS-12637. Extend TestDistributedFileSystemWithECFile with a random EC policy. Contributed by Takanobu Asanuma.
Date Mon, 16 Oct 2017 16:55:33 GMT
Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 77ef81055 -> ca1271e21


HDFS-12637. Extend TestDistributedFileSystemWithECFile with a random EC policy. Contributed
by Takanobu Asanuma.

(cherry picked from commit f57e5aae77a004ac9a8d548cf7fa2669e3642f30)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca1271e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca1271e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca1271e2

Branch: refs/heads/branch-3.0
Commit: ca1271e2180b7573a2bae5753f7a6055918872da
Parents: 77ef810
Author: Xiao Chen <xiao@apache.org>
Authored: Mon Oct 16 09:54:37 2017 -0700
Committer: Xiao Chen <xiao@apache.org>
Committed: Mon Oct 16 09:55:31 2017 -0700

----------------------------------------------------------------------
 .../TestDistributedFileSystemWithECFile.java    | 37 +++++++++------
 ...dFileSystemWithECFileWithRandomECPolicy.java | 49 ++++++++++++++++++++
 2 files changed, 73 insertions(+), 13 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca1271e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
index d4e01b7..14a2ec4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFile.java
@@ -39,34 +39,45 @@ import static org.junit.Assert.assertTrue;
  * FileSystem.listFiles for erasure coded files.
  */
 public class TestDistributedFileSystemWithECFile {
-  private final ErasureCodingPolicy ecPolicy =
-      StripedFileTestUtil.getDefaultECPolicy();
-  private final int cellSize = ecPolicy.getCellSize();
-  private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
-  private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
-  private final int numDNs = dataBlocks + parityBlocks;
-  private final int stripesPerBlock = 4;
-  private final int blockSize = stripesPerBlock * cellSize;
-  private final int blockGroupSize = blockSize * dataBlocks;
+  private ErasureCodingPolicy ecPolicy;
+  private int cellSize;
+  private short dataBlocks;
+  private short parityBlocks;
+  private int numDNs;
+  private int stripesPerBlock;
+  private int blockSize;
+  private int blockGroupSize;
 
   private MiniDFSCluster cluster;
   private FileContext fileContext;
   private DistributedFileSystem fs;
   private Configuration conf = new HdfsConfiguration();
 
+  public ErasureCodingPolicy getEcPolicy() {
+    return StripedFileTestUtil.getDefaultECPolicy();
+  }
+
   @Before
   public void setup() throws IOException {
+    ecPolicy = getEcPolicy();
+    cellSize = ecPolicy.getCellSize();
+    dataBlocks = (short) ecPolicy.getNumDataUnits();
+    parityBlocks = (short) ecPolicy.getNumParityUnits();
+    numDNs = dataBlocks + parityBlocks;
+    stripesPerBlock = 4;
+    blockSize = stripesPerBlock * cellSize;
+    blockGroupSize = blockSize * dataBlocks;
+
     conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
     conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY,
         false);
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
     fileContext = FileContext.getFileContext(cluster.getURI(0), conf);
     fs = cluster.getFileSystem();
-    fs.enableErasureCodingPolicy(
-        StripedFileTestUtil.getDefaultECPolicy().getName());
+    fs.enableErasureCodingPolicy(ecPolicy.getName());
     fs.mkdirs(new Path("/ec"));
     cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec",
-        StripedFileTestUtil.getDefaultECPolicy().getName());
+        ecPolicy.getName());
   }
 
   @After
@@ -121,7 +132,7 @@ public class TestDistributedFileSystemWithECFile {
 
   @Test(timeout=60000)
   public void testListECFilesSmallerThanOneStripe() throws Exception {
-    int dataBlocksNum = 3;
+    int dataBlocksNum = dataBlocks;
     createFile("/ec/smallstripe", cellSize * dataBlocksNum);
     RemoteIterator<LocatedFileStatus> iter =
         cluster.getFileSystem().listFiles(new Path("/ec"), true);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca1271e2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java
new file mode 100644
index 0000000..afa7569
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystemWithECFileWithRandomECPolicy.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This test extends TestDistributedFileSystemWithECFile to use a random
+ * (non-default) EC policy.
+ */
+public class TestDistributedFileSystemWithECFileWithRandomECPolicy extends
+    TestDistributedFileSystemWithECFile {
+  private static final Logger LOG = LoggerFactory.getLogger(
+      TestDistributedFileSystemWithECFileWithRandomECPolicy.class);
+
+  private ErasureCodingPolicy ecPolicy;
+
+  public TestDistributedFileSystemWithECFileWithRandomECPolicy() {
+    // If you want to debug this test with a specific ec policy, please use
+    // SystemErasureCodingPolicies class.
+    // e.g. ecPolicy = SystemErasureCodingPolicies.getByID(RS_3_2_POLICY_ID);
+    ecPolicy = StripedFileTestUtil.getRandomNonDefaultECPolicy();
+    LOG.info("run {} with {}.",
+        TestDistributedFileSystemWithECFileWithRandomECPolicy.class
+            .getSuperclass().getSimpleName(), ecPolicy.getName());
+  }
+
+  @Override
+  public ErasureCodingPolicy getEcPolicy() {
+    return ecPolicy;
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message