hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From su...@apache.org
Subject [01/50] [abbrv] hadoop git commit: HDFS-11117. Refactor striped file tests to allow flexibly test erasure coding policy. Contributed by Sammi Chen [Forced Update!]
Date Wed, 23 Nov 2016 02:38:36 GMT
Repository: hadoop
Updated Branches:
  refs/heads/YARN-2915 b91e900cc -> 6452592b8 (forced update)


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
index 13dcccf..15a8756 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.StripedFileTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
@@ -54,13 +55,15 @@ public class TestAddOverReplicatedStripedBlocks {
   private DistributedFileSystem fs;
   private final Path dirPath = new Path("/striped");
   private Path filePath = new Path(dirPath, "file");
-  private final short DATA_BLK_NUM = StripedFileTestUtil.NUM_DATA_BLOCKS;
-  private final short PARITY_BLK_NUM = StripedFileTestUtil.NUM_PARITY_BLOCKS;
-  private final short GROUP_SIZE = (short) (DATA_BLK_NUM + PARITY_BLK_NUM);
-  private final int CELLSIZE = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-  private final int NUM_STRIPE_PER_BLOCK = 4;
-  private final int BLOCK_SIZE = NUM_STRIPE_PER_BLOCK * CELLSIZE;
-  private final int numDNs = GROUP_SIZE + 3;
+  private final ErasureCodingPolicy ecPolicy =
+      ErasureCodingPolicyManager.getSystemDefaultPolicy();
+  private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+  private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+  private final short groupSize = (short) (dataBlocks + parityBlocks);
+  private final int cellSize = ecPolicy.getCellSize();
+  private final int stripesPerBlock = 4;
+  private final int blockSize = stripesPerBlock * cellSize;
+  private final int numDNs = groupSize + 3;
 
   @Rule
   public Timeout globalTimeout = new Timeout(300000);
@@ -68,7 +71,7 @@ public class TestAddOverReplicatedStripedBlocks {
   @Before
   public void setup() throws IOException {
     Configuration conf = new Configuration();
-    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
     // disable block recovery
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
@@ -92,17 +95,17 @@ public class TestAddOverReplicatedStripedBlocks {
   @Test
   public void testProcessOverReplicatedStripedBlock() throws Exception {
     // create a file which has exact one block group to the first GROUP_SIZE DNs
-    long fileLen = DATA_BLK_NUM * BLOCK_SIZE;
+    long fileLen = dataBlocks * blockSize;
     DFSTestUtil.createStripedFile(cluster, filePath, null, 1,
-        NUM_STRIPE_PER_BLOCK, false);
+        stripesPerBlock, false);
     LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
         filePath.toString(), 0, fileLen);
     LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
     long gs = bg.getBlock().getGenerationStamp();
     String bpid = bg.getBlock().getBlockPoolId();
     long groupId = bg.getBlock().getBlockId();
-    Block blk = new Block(groupId, BLOCK_SIZE, gs);
-    for (int i = 0; i < GROUP_SIZE; i++) {
+    Block blk = new Block(groupId, blockSize, gs);
+    for (int i = 0; i < groupSize; i++) {
       blk.setBlockId(groupId + i);
       cluster.injectBlocks(i, Arrays.asList(blk), bpid);
     }
@@ -113,7 +116,7 @@ public class TestAddOverReplicatedStripedBlocks {
     cluster.injectBlocks(numDNs - 3, Arrays.asList(blk), bpid);
     cluster.injectBlocks(numDNs - 2, Arrays.asList(blk), bpid);
     // let a internal block be over replicated with 1 redundant block.
-    blk.setBlockId(groupId + DATA_BLK_NUM);
+    blk.setBlockId(groupId + dataBlocks);
     cluster.injectBlocks(numDNs - 1, Arrays.asList(blk), bpid);
 
     // update blocksMap
@@ -128,14 +131,14 @@ public class TestAddOverReplicatedStripedBlocks {
     // verify that all internal blocks exists
     lbs = cluster.getNameNodeRpc().getBlockLocations(
         filePath.toString(), 0, fileLen);
-    StripedFileTestUtil.verifyLocatedStripedBlocks(lbs, GROUP_SIZE);
+    StripedFileTestUtil.verifyLocatedStripedBlocks(lbs, groupSize);
   }
 
   @Test
   public void testProcessOverReplicatedSBSmallerThanFullBlocks()
       throws Exception {
     // Create a EC file which doesn't fill full internal blocks.
-    int fileLen = CELLSIZE * (DATA_BLK_NUM - 1);
+    int fileLen = cellSize * (dataBlocks - 1);
     byte[] content = new byte[fileLen];
     DFSTestUtil.writeFile(fs, filePath, new String(content));
     LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
@@ -144,7 +147,7 @@ public class TestAddOverReplicatedStripedBlocks {
     long gs = bg.getBlock().getGenerationStamp();
     String bpid = bg.getBlock().getBlockPoolId();
     long groupId = bg.getBlock().getBlockId();
-    Block blk = new Block(groupId, BLOCK_SIZE, gs);
+    Block blk = new Block(groupId, blockSize, gs);
     cluster.triggerBlockReports();
     List<DatanodeInfo> infos = Arrays.asList(bg.getLocations());
 
@@ -171,25 +174,25 @@ public class TestAddOverReplicatedStripedBlocks {
     // verify that all internal blocks exists
     lbs = cluster.getNameNodeRpc().getBlockLocations(
         filePath.toString(), 0, fileLen);
-    StripedFileTestUtil.verifyLocatedStripedBlocks(lbs, GROUP_SIZE - 1);
+    StripedFileTestUtil.verifyLocatedStripedBlocks(lbs, groupSize - 1);
   }
 
   @Test
   public void testProcessOverReplicatedAndCorruptStripedBlock()
       throws Exception {
-    long fileLen = DATA_BLK_NUM * BLOCK_SIZE;
+    long fileLen = dataBlocks * blockSize;
     DFSTestUtil.createStripedFile(cluster, filePath, null, 1,
-        NUM_STRIPE_PER_BLOCK, false);
+        stripesPerBlock, false);
     LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
         filePath.toString(), 0, fileLen);
     LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
     long gs = bg.getBlock().getGenerationStamp();
     String bpid = bg.getBlock().getBlockPoolId();
     long groupId = bg.getBlock().getBlockId();
-    Block blk = new Block(groupId, BLOCK_SIZE, gs);
+    Block blk = new Block(groupId, blockSize, gs);
     BlockInfoStriped blockInfo = new BlockInfoStriped(blk,
         ErasureCodingPolicyManager.getSystemDefaultPolicy());
-    for (int i = 0; i < GROUP_SIZE; i++) {
+    for (int i = 0; i < groupSize; i++) {
       blk.setBlockId(groupId + i);
       cluster.injectBlocks(i, Arrays.asList(blk), bpid);
     }
@@ -225,14 +228,14 @@ public class TestAddOverReplicatedStripedBlocks {
     lbs = cluster.getNameNodeRpc().getBlockLocations(filePath.toString(), 0,
         fileLen);
     bg = (LocatedStripedBlock) (lbs.get(0));
-    assertEquals(GROUP_SIZE + 1, bg.getBlockIndices().length);
-    assertEquals(GROUP_SIZE + 1, bg.getLocations().length);
-    BitSet set = new BitSet(GROUP_SIZE);
+    assertEquals(groupSize + 1, bg.getBlockIndices().length);
+    assertEquals(groupSize + 1, bg.getLocations().length);
+    BitSet set = new BitSet(groupSize);
     for (byte index : bg.getBlockIndices()) {
       set.set(index);
     }
     Assert.assertFalse(set.get(0));
-    for (int i = 1; i < GROUP_SIZE; i++) {
+    for (int i = 1; i < groupSize; i++) {
       assertTrue(set.get(i));
     }
   }
@@ -243,18 +246,18 @@ public class TestAddOverReplicatedStripedBlocks {
   @Test
   public void testProcessOverReplicatedAndMissingStripedBlock()
       throws Exception {
-    long fileLen = CELLSIZE * DATA_BLK_NUM;
+    long fileLen = cellSize * dataBlocks;
     DFSTestUtil.createStripedFile(cluster, filePath, null, 1,
-        NUM_STRIPE_PER_BLOCK, false);
+        stripesPerBlock, false);
     LocatedBlocks lbs = cluster.getNameNodeRpc().getBlockLocations(
         filePath.toString(), 0, fileLen);
     LocatedStripedBlock bg = (LocatedStripedBlock) (lbs.get(0));
     long gs = bg.getBlock().getGenerationStamp();
     String bpid = bg.getBlock().getBlockPoolId();
     long groupId = bg.getBlock().getBlockId();
-    Block blk = new Block(groupId, BLOCK_SIZE, gs);
+    Block blk = new Block(groupId, blockSize, gs);
     // only inject GROUP_SIZE - 1 blocks, so there is one block missing
-    for (int i = 0; i < GROUP_SIZE - 1; i++) {
+    for (int i = 0; i < groupSize - 1; i++) {
       blk.setBlockId(groupId + i);
       cluster.injectBlocks(i, Arrays.asList(blk), bpid);
     }
@@ -282,14 +285,14 @@ public class TestAddOverReplicatedStripedBlocks {
     lbs = cluster.getNameNodeRpc().getBlockLocations(filePath.toString(), 0,
         fileLen);
     bg = (LocatedStripedBlock) (lbs.get(0));
-    assertEquals(GROUP_SIZE + 1, bg.getBlockIndices().length);
-    assertEquals(GROUP_SIZE + 1, bg.getLocations().length);
-    BitSet set = new BitSet(GROUP_SIZE);
+    assertEquals(groupSize + 1, bg.getBlockIndices().length);
+    assertEquals(groupSize + 1, bg.getLocations().length);
+    BitSet set = new BitSet(groupSize);
     for (byte index : bg.getBlockIndices()) {
       set.set(index);
     }
-    Assert.assertFalse(set.get(GROUP_SIZE - 1));
-    for (int i = 0; i < GROUP_SIZE - 1; i++) {
+    Assert.assertFalse(set.get(groupSize - 1));
+    for (int i = 0; i < groupSize - 1; i++) {
       assertTrue(set.get(i));
     }
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
index 37b334f..ab24a25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
 import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
@@ -41,12 +42,13 @@ import org.mockito.internal.util.reflection.Whitebox;
 
 import java.io.IOException;
 
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
-
 public class TestAddStripedBlockInFBR {
-  private final short GROUP_SIZE = (short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS);
+  private final ErasureCodingPolicy ecPolicy =
+      ErasureCodingPolicyManager.getSystemDefaultPolicy();
+  private final int cellSize = ecPolicy.getCellSize();
+  private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+  private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+  private final short groupSize = (short) (dataBlocks + parityBlocks);
 
   private MiniDFSCluster cluster;
   private DistributedFileSystem dfs;
@@ -57,7 +59,7 @@ public class TestAddStripedBlockInFBR {
   @Before
   public void setup() throws IOException {
     Configuration conf = new HdfsConfiguration();
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
     cluster.waitActive();
     dfs = cluster.getFileSystem();
   }
@@ -87,14 +89,14 @@ public class TestAddStripedBlockInFBR {
     dfs.getClient().setErasureCodingPolicy(ecDir.toString(), null);
 
     // create several non-EC files and one EC file
-    final Path[] repFiles = new Path[GROUP_SIZE];
-    for (int i = 0; i < GROUP_SIZE; i++) {
+    final Path[] repFiles = new Path[groupSize];
+    for (int i = 0; i < groupSize; i++) {
       repFiles[i] = new Path(repDir, "f" + i);
       DFSTestUtil.createFile(dfs, repFiles[i], 1L, (short) 3, 0L);
     }
     final Path ecFile = new Path(ecDir, "f");
     DFSTestUtil.createFile(dfs, ecFile,
-        BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS, (short) 1, 0L);
+        cellSize * dataBlocks, (short) 1, 0L);
 
     // trigger dn's FBR. The FBR will add block-dn mapping.
     DataNodeTestUtils.triggerBlockReport(dn);
@@ -103,7 +105,7 @@ public class TestAddStripedBlockInFBR {
     BlockInfoStriped blockInfo = (BlockInfoStriped) cluster.getNamesystem()
         .getFSDirectory().getINode(ecFile.toString()).asFile().getLastBlock();
     NumberReplicas nr = spy.countNodes(blockInfo);
-    Assert.assertEquals(GROUP_SIZE, nr.liveReplicas());
+    Assert.assertEquals(groupSize, nr.liveReplicas());
     Assert.assertEquals(0, nr.excessReplicas());
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
index 768533a..ffd3fa7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
@@ -24,11 +24,11 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
 import org.apache.hadoop.hdfs.protocol.DatanodeID;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -63,13 +63,16 @@ import java.util.List;
 import java.util.UUID;
 
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
 import static org.junit.Assert.assertEquals;
 
 public class TestAddStripedBlocks {
-  private final short GROUP_SIZE = (short) (StripedFileTestUtil.NUM_DATA_BLOCKS +
-      StripedFileTestUtil.NUM_PARITY_BLOCKS);
+  private final ErasureCodingPolicy ecPolicy =
+      ErasureCodingPolicyManager.getSystemDefaultPolicy();
+  private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+  private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+  private final int cellSize = ecPolicy.getCellSize();
+  private final short groupSize = (short) (ecPolicy.getNumDataUnits() +
+      ecPolicy.getNumParityUnits());
 
   private MiniDFSCluster cluster;
   private DistributedFileSystem dfs;
@@ -80,7 +83,7 @@ public class TestAddStripedBlocks {
   @Before
   public void setup() throws IOException {
     cluster = new MiniDFSCluster.Builder(new HdfsConfiguration())
-        .numDataNodes(GROUP_SIZE).build();
+        .numDataNodes(groupSize).build();
     cluster.waitActive();
     dfs = cluster.getFileSystem();
     dfs.getClient().setErasureCodingPolicy("/", null);
@@ -206,16 +209,15 @@ public class TestAddStripedBlocks {
       boolean checkReplica) {
     assertEquals(0, block.numNodes());
     Assert.assertFalse(block.isComplete());
-    Assert.assertEquals(StripedFileTestUtil.NUM_DATA_BLOCKS, block.getDataBlockNum());
-    Assert.assertEquals(StripedFileTestUtil.NUM_PARITY_BLOCKS,
-        block.getParityBlockNum());
+    Assert.assertEquals(dataBlocks, block.getDataBlockNum());
+    Assert.assertEquals(parityBlocks, block.getParityBlockNum());
     Assert.assertEquals(0,
         block.getBlockId() & HdfsServerConstants.BLOCK_GROUP_INDEX_MASK);
 
     Assert.assertEquals(HdfsServerConstants.BlockUCState.UNDER_CONSTRUCTION,
         block.getBlockUCState());
     if (checkReplica) {
-      Assert.assertEquals(GROUP_SIZE,
+      Assert.assertEquals(groupSize,
           block.getUnderConstructionFeature().getNumExpectedLocations());
       DatanodeStorageInfo[] storages = block.getUnderConstructionFeature()
           .getExpectedStorageLocations();
@@ -259,8 +261,8 @@ public class TestAddStripedBlocks {
       Assert.assertTrue(lblk instanceof LocatedStripedBlock);
       DatanodeInfo[] datanodes = lblk.getLocations();
       byte[] blockIndices = ((LocatedStripedBlock) lblk).getBlockIndices();
-      Assert.assertEquals(GROUP_SIZE, datanodes.length);
-      Assert.assertEquals(GROUP_SIZE, blockIndices.length);
+      Assert.assertEquals(groupSize, datanodes.length);
+      Assert.assertEquals(groupSize, blockIndices.length);
       Assert.assertArrayEquals(indices, blockIndices);
       Assert.assertArrayEquals(expectedDNs, datanodes);
     } finally {
@@ -291,8 +293,8 @@ public class TestAddStripedBlocks {
       DatanodeStorageInfo[] locs = lastBlock.getUnderConstructionFeature()
           .getExpectedStorageLocations();
       byte[] indices = lastBlock.getUnderConstructionFeature().getBlockIndices();
-      Assert.assertEquals(GROUP_SIZE, locs.length);
-      Assert.assertEquals(GROUP_SIZE, indices.length);
+      Assert.assertEquals(groupSize, locs.length);
+      Assert.assertEquals(groupSize, indices.length);
 
       // 2. mimic incremental block reports and make sure the uc-replica list in
       // the BlockInfoUCStriped is correct
@@ -314,8 +316,8 @@ public class TestAddStripedBlocks {
       // make sure lastBlock is correct and the storages have been updated
       locs = lastBlock.getUnderConstructionFeature().getExpectedStorageLocations();
       indices = lastBlock.getUnderConstructionFeature().getBlockIndices();
-      Assert.assertEquals(GROUP_SIZE, locs.length);
-      Assert.assertEquals(GROUP_SIZE, indices.length);
+      Assert.assertEquals(groupSize, locs.length);
+      Assert.assertEquals(groupSize, indices.length);
       for (DatanodeStorageInfo newstorage : locs) {
         Assert.assertTrue(storageIDs.contains(newstorage.getStorageID()));
       }
@@ -330,7 +332,7 @@ public class TestAddStripedBlocks {
     INodeFile fileNode = cluster.getNamesystem().getFSDirectory()
         .getINode4Write(file.toString()).asFile();
     BlockInfo lastBlock = fileNode.getLastBlock();
-    int i = GROUP_SIZE - 1;
+    int i = groupSize - 1;
     for (DataNode dn : cluster.getDataNodes()) {
       String storageID = storageIDs.get(i);
       final Block block = new Block(lastBlock.getBlockId() + i--,
@@ -351,12 +353,12 @@ public class TestAddStripedBlocks {
     DatanodeStorageInfo[] locs = lastBlock.getUnderConstructionFeature()
         .getExpectedStorageLocations();
     byte[] indices = lastBlock.getUnderConstructionFeature().getBlockIndices();
-    Assert.assertEquals(GROUP_SIZE, locs.length);
-    Assert.assertEquals(GROUP_SIZE, indices.length);
-    for (i = 0; i < GROUP_SIZE; i++) {
+    Assert.assertEquals(groupSize, locs.length);
+    Assert.assertEquals(groupSize, indices.length);
+    for (i = 0; i < groupSize; i++) {
       Assert.assertEquals(storageIDs.get(i),
-          locs[GROUP_SIZE - 1 - i].getStorageID());
-      Assert.assertEquals(GROUP_SIZE - i - 1, indices[i]);
+          locs[groupSize - 1 - i].getStorageID());
+      Assert.assertEquals(groupSize - i - 1, indices[i]);
     }
   }
 
@@ -380,7 +382,7 @@ public class TestAddStripedBlocks {
     // Now send a block report with correct size
     DatanodeStorage storage = new DatanodeStorage(UUID.randomUUID().toString());
     final Block reported = new Block(stored);
-    reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE);
+    reported.setNumBytes(numStripes * cellSize);
     StorageReceivedDeletedBlocks[] reports = DFSTestUtil
         .makeReportForReceivedBlock(reported,
             ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
@@ -391,7 +393,7 @@ public class TestAddStripedBlocks {
 
     // Now send a block report with wrong size
     reported.setBlockId(stored.getBlockId() + 1);
-    reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE - 1);
+    reported.setNumBytes(numStripes * cellSize - 1);
     reports = DFSTestUtil.makeReportForReceivedBlock(reported,
             ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
     ns.processIncrementalBlockReport(
@@ -400,8 +402,8 @@ public class TestAddStripedBlocks {
     Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
 
     // Now send a parity block report with correct size
-    reported.setBlockId(stored.getBlockId() + NUM_DATA_BLOCKS);
-    reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE);
+    reported.setBlockId(stored.getBlockId() + dataBlocks);
+    reported.setNumBytes(numStripes * cellSize);
     reports = DFSTestUtil.makeReportForReceivedBlock(reported,
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
     ns.processIncrementalBlockReport(
@@ -410,8 +412,8 @@ public class TestAddStripedBlocks {
     Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
 
     // Now send a parity block report with wrong size
-    reported.setBlockId(stored.getBlockId() + NUM_DATA_BLOCKS);
-    reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE + 1);
+    reported.setBlockId(stored.getBlockId() + dataBlocks);
+    reported.setNumBytes(numStripes * cellSize + 1);
     reports = DFSTestUtil.makeReportForReceivedBlock(reported,
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
     ns.processIncrementalBlockReport(
@@ -425,8 +427,8 @@ public class TestAddStripedBlocks {
     // Now change the size of stored block, and test verifying the last
     // block size
     stored.setNumBytes(stored.getNumBytes() + 10);
-    reported.setBlockId(stored.getBlockId() + NUM_DATA_BLOCKS + 2);
-    reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE);
+    reported.setBlockId(stored.getBlockId() + dataBlocks + 2);
+    reported.setNumBytes(numStripes * cellSize);
     reports = DFSTestUtil.makeReportForReceivedBlock(reported,
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
     ns.processIncrementalBlockReport(
@@ -438,9 +440,9 @@ public class TestAddStripedBlocks {
     // Now send a parity block report with correct size based on adjusted
     // size of stored block
     /** Now stored block has {@link numStripes} full stripes + a cell + 10 */
-    stored.setNumBytes(stored.getNumBytes() + BLOCK_STRIPED_CELL_SIZE);
+    stored.setNumBytes(stored.getNumBytes() + cellSize);
     reported.setBlockId(stored.getBlockId());
-    reported.setNumBytes((numStripes + 1) * BLOCK_STRIPED_CELL_SIZE);
+    reported.setNumBytes((numStripes + 1) * cellSize);
     reports = DFSTestUtil.makeReportForReceivedBlock(reported,
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
     ns.processIncrementalBlockReport(
@@ -450,7 +452,7 @@ public class TestAddStripedBlocks {
     Assert.assertEquals(3, bm.getCorruptReplicas(stored).size());
 
     reported.setBlockId(stored.getBlockId() + 1);
-    reported.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE + 10);
+    reported.setNumBytes(numStripes * cellSize + 10);
     reports = DFSTestUtil.makeReportForReceivedBlock(reported,
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
     ns.processIncrementalBlockReport(
@@ -459,8 +461,8 @@ public class TestAddStripedBlocks {
     Assert.assertEquals(1, ns.getCorruptReplicaBlocks());
     Assert.assertEquals(3, bm.getCorruptReplicas(stored).size());
 
-    reported.setBlockId(stored.getBlockId() + NUM_DATA_BLOCKS);
-    reported.setNumBytes((numStripes + 1) * BLOCK_STRIPED_CELL_SIZE);
+    reported.setBlockId(stored.getBlockId() + dataBlocks);
+    reported.setNumBytes((numStripes + 1) * cellSize);
     reports = DFSTestUtil.makeReportForReceivedBlock(reported,
         ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, storage);
     ns.processIncrementalBlockReport(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
index fe29e1c..f6e99ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
@@ -99,8 +98,8 @@ public class TestFSEditLogLoader {
 
   private static final int NUM_DATA_NODES = 0;
 
-  private static final ErasureCodingPolicy testECPolicy
-      = StripedFileTestUtil.TEST_EC_POLICY;
+  private final ErasureCodingPolicy testECPolicy
+      = ErasureCodingPolicyManager.getSystemDefaultPolicy();
 
   @Test
   public void testDisplayRecentEditLogOpCodes() throws IOException {
@@ -474,8 +473,8 @@ public class TestFSEditLogLoader {
       long blkId = 1;
       long blkNumBytes = 1024;
       long timestamp = 1426222918;
-      short blockNum = StripedFileTestUtil.NUM_DATA_BLOCKS;
-      short parityNum = StripedFileTestUtil.NUM_PARITY_BLOCKS;
+      short blockNum = (short) testECPolicy.getNumDataUnits();
+      short parityNum = (short) testECPolicy.getNumParityUnits();
 
       //set the storage policy of the directory
       fs.mkdir(new Path(testDir), new FsPermission("755"));
@@ -547,8 +546,8 @@ public class TestFSEditLogLoader {
       long blkId = 1;
       long blkNumBytes = 1024;
       long timestamp = 1426222918;
-      short blockNum = StripedFileTestUtil.NUM_DATA_BLOCKS;
-      short parityNum = StripedFileTestUtil.NUM_PARITY_BLOCKS;
+      short blockNum = (short) testECPolicy.getNumDataUnits();
+      short parityNum = (short) testECPolicy.getNumParityUnits();
 
       //set the storage policy of the directory
       fs.mkdir(new Path(testDir), new FsPermission("755"));

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
index b153652..9ec34d2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java
@@ -25,7 +25,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -46,11 +45,12 @@ import java.io.IOException;
 public class TestQuotaWithStripedBlocks {
   private static final int BLOCK_SIZE = 1024 * 1024;
   private static final long DISK_QUOTA = BLOCK_SIZE * 10;
-  private static final ErasureCodingPolicy ecPolicy =
+  private final ErasureCodingPolicy ecPolicy =
       ErasureCodingPolicyManager.getSystemDefaultPolicy();
-  private static final int NUM_DATA_BLOCKS = ecPolicy.getNumDataUnits();
-  private static final int NUM_PARITY_BLOCKS = ecPolicy.getNumParityUnits();
-  private static final int GROUP_SIZE = NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS;
+  private final int dataBlocks = ecPolicy.getNumDataUnits();
+  private final int parityBlocsk = ecPolicy.getNumParityUnits();
+  private final int groupSize = dataBlocks + parityBlocsk;
+  private final int cellSize = ecPolicy.getCellSize();
   private static final Path ecDir = new Path("/ec");
 
   private MiniDFSCluster cluster;
@@ -64,7 +64,7 @@ public class TestQuotaWithStripedBlocks {
   public void setUp() throws IOException {
     final Configuration conf = new Configuration();
     conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize).build();
     cluster.waitActive();
 
     dir = cluster.getNamesystem().getFSDirectory();
@@ -109,8 +109,8 @@ public class TestQuotaWithStripedBlocks {
       final long diskUsed = dirNode.getDirectoryWithQuotaFeature()
           .getSpaceConsumed().getTypeSpaces().get(StorageType.DISK);
       // When we add a new block we update the quota using the full block size.
-      Assert.assertEquals(BLOCK_SIZE * GROUP_SIZE, spaceUsed);
-      Assert.assertEquals(BLOCK_SIZE * GROUP_SIZE, diskUsed);
+      Assert.assertEquals(BLOCK_SIZE * groupSize, spaceUsed);
+      Assert.assertEquals(BLOCK_SIZE * groupSize, diskUsed);
 
       dfs.getClient().getNamenode().complete(file.toString(),
           dfs.getClient().getClientName(), previous, fileNode.getId());
@@ -120,9 +120,9 @@ public class TestQuotaWithStripedBlocks {
       final long actualDiskUsed = dirNode.getDirectoryWithQuotaFeature()
           .getSpaceConsumed().getTypeSpaces().get(StorageType.DISK);
       // In this case the file's real size is cell size * block group size.
-      Assert.assertEquals(StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE * GROUP_SIZE,
+      Assert.assertEquals(cellSize * groupSize,
           actualSpaceUsed);
-      Assert.assertEquals(StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE * GROUP_SIZE,
+      Assert.assertEquals(cellSize * groupSize,
           actualDiskUsed);
     } finally {
       IOUtils.cleanup(null, out);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
index c38b6b4..d43804a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.StripedFileTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
 import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
@@ -51,9 +52,6 @@ import org.slf4j.LoggerFactory;
 import java.util.BitSet;
 import java.util.List;
 
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
-import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -61,9 +59,14 @@ import static org.junit.Assert.assertTrue;
 public class TestReconstructStripedBlocks {
   public static final Logger LOG = LoggerFactory.getLogger(
       TestReconstructStripedBlocks.class);
-  private static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-  private final short GROUP_SIZE =
-      (short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS);
+  private final ErasureCodingPolicy ecPolicy =
+      ErasureCodingPolicyManager.getSystemDefaultPolicy();
+  private final int cellSize = ecPolicy.getCellSize();
+  private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+  private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+  private final short groupSize = (short) (dataBlocks + parityBlocks);
+  private final int blockSize = 4 * cellSize;
+
 
   private MiniDFSCluster cluster;
   private final Path dirPath = new Path("/dir");
@@ -88,7 +91,7 @@ public class TestReconstructStripedBlocks {
 
   @Test
   public void testMissingStripedBlockWithBusyNode() throws Exception {
-    for (int i = 1; i <= NUM_PARITY_BLOCKS; i++) {
+    for (int i = 1; i <= parityBlocks; i++) {
       doTestMissingStripedBlock(i, 1);
     }
   }
@@ -105,7 +108,7 @@ public class TestReconstructStripedBlocks {
       throws Exception {
     Configuration conf = new HdfsConfiguration();
     initConf(conf);
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE + 1)
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 1)
         .build();
 
     try {
@@ -126,10 +129,10 @@ public class TestReconstructStripedBlocks {
       for (BlockInfo blk : blocks) {
         assertTrue(blk.isStriped());
         assertTrue(blk.isComplete());
-        assertEquals(BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS,
+        assertEquals(cellSize * dataBlocks,
             blk.getNumBytes());
         final BlockInfoStriped sb = (BlockInfoStriped) blk;
-        assertEquals(GROUP_SIZE, sb.numNodes());
+        assertEquals(groupSize, sb.numNodes());
       }
 
       final BlockManager bm = cluster.getNamesystem().getBlockManager();
@@ -156,7 +159,7 @@ public class TestReconstructStripedBlocks {
       BlockManagerTestUtil.getComputedDatanodeWork(bm);
 
       // all the reconstruction work will be scheduled on the last DN
-      DataNode lastDn = cluster.getDataNodes().get(GROUP_SIZE);
+      DataNode lastDn = cluster.getDataNodes().get(groupSize);
       DatanodeDescriptor last =
           bm.getDatanodeManager().getDatanode(lastDn.getDatanodeId());
       assertEquals("Counting the number of outstanding EC tasks", numBlocks,
@@ -168,15 +171,15 @@ public class TestReconstructStripedBlocks {
         assertEquals(last, info.getTargetDnInfos()[0]);
         assertEquals(info.getSourceDnInfos().length,
             info.getLiveBlockIndices().length);
-        if (GROUP_SIZE - numOfMissed == NUM_DATA_BLOCKS) {
+        if (groupSize - numOfMissed == dataBlocks) {
           // It's a QUEUE_HIGHEST_PRIORITY block, so the busy DNs will be chosen
           // to make sure we have NUM_DATA_BLOCKS DNs to do reconstruction
           // work.
-          assertEquals(NUM_DATA_BLOCKS, info.getSourceDnInfos().length);
+          assertEquals(dataBlocks, info.getSourceDnInfos().length);
         } else {
           // The block has no highest priority, so we don't use the busy DNs as
           // sources
-          assertEquals(GROUP_SIZE - numOfMissed - numOfBusy,
+          assertEquals(groupSize - numOfMissed - numOfBusy,
               info.getSourceDnInfos().length);
         }
       }
@@ -190,15 +193,15 @@ public class TestReconstructStripedBlocks {
     Configuration conf = new HdfsConfiguration();
     conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1000);
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1000);
-    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, StripedFileTestUtil.blockSize);
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE + 2)
+    conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2)
         .build();
     try {
       cluster.waitActive();
       DistributedFileSystem fs = cluster.getFileSystem();
       BlockManager bm = cluster.getNamesystem().getBlockManager();
       fs.getClient().setErasureCodingPolicy("/", null);
-      int fileLen = NUM_DATA_BLOCKS * StripedFileTestUtil.blockSize;
+      int fileLen = dataBlocks * blockSize;
       Path p = new Path("/test2RecoveryTasksForSameBlockGroup");
       final byte[] data = new byte[fileLen];
       DFSTestUtil.writeFile(fs, p, data);
@@ -206,7 +209,7 @@ public class TestReconstructStripedBlocks {
       LocatedStripedBlock lb = (LocatedStripedBlock)fs.getClient()
           .getLocatedBlocks(p.toString(), 0).get(0);
       LocatedBlock[] lbs = StripedBlockUtil.parseStripedBlockGroup(lb,
-          cellSize, NUM_DATA_BLOCKS, NUM_PARITY_BLOCKS);
+          cellSize, dataBlocks, parityBlocks);
 
       assertEquals(0, getNumberOfBlocksToBeErasureCoded(cluster));
       assertEquals(0, bm.getPendingReconstructionBlocksCount());
@@ -255,7 +258,7 @@ public class TestReconstructStripedBlocks {
     conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1);
     conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REPLICATION_CONSIDERLOAD_KEY,
         false);
-    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE + 2)
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(groupSize + 2)
         .build();
     cluster.waitActive();
     DistributedFileSystem fs = cluster.getFileSystem();
@@ -264,7 +267,7 @@ public class TestReconstructStripedBlocks {
       fs.mkdirs(dirPath);
       fs.setErasureCodingPolicy(dirPath, null);
       DFSTestUtil.createFile(fs, filePath,
-          BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS * 2, (short) 1, 0L);
+          cellSize * dataBlocks * 2, (short) 1, 0L);
 
       // stop a dn
       LocatedBlocks blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
@@ -275,7 +278,7 @@ public class TestReconstructStripedBlocks {
       cluster.setDataNodeDead(dnToStop);
 
       // wait for reconstruction to happen
-      DFSTestUtil.waitForReplication(fs, filePath, GROUP_SIZE, 15 * 1000);
+      DFSTestUtil.waitForReplication(fs, filePath, groupSize, 15 * 1000);
 
       // bring the dn back: 10 internal blocks now
       cluster.restartDataNode(dnProp);
@@ -304,7 +307,7 @@ public class TestReconstructStripedBlocks {
       // check if NN can detect the missing internal block and finish the
       // reconstruction
       StripedFileTestUtil.waitForReconstructionFinished(filePath, fs,
-          GROUP_SIZE);
+          groupSize);
       boolean reconstructed = false;
       for (int i = 0; i < 5; i++) {
         NumberReplicas num = null;
@@ -316,7 +319,7 @@ public class TestReconstructStripedBlocks {
         } finally {
           fsn.readUnlock();
         }
-        if (num.liveReplicas() >= GROUP_SIZE) {
+        if (num.liveReplicas() >= groupSize) {
           reconstructed = true;
           break;
         } else {
@@ -327,11 +330,11 @@ public class TestReconstructStripedBlocks {
 
       blks = fs.getClient().getLocatedBlocks(filePath.toString(), 0);
       block = (LocatedStripedBlock) blks.getLastLocatedBlock();
-      BitSet bitSet = new BitSet(GROUP_SIZE);
+      BitSet bitSet = new BitSet(groupSize);
       for (byte index : block.getBlockIndices()) {
         bitSet.set(index);
       }
-      for (int i = 0; i < GROUP_SIZE; i++) {
+      for (int i = 0; i < groupSize; i++) {
         Assert.assertTrue(bitSet.get(i));
       }
     } finally {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
index 3703501..6c4d73c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
 import org.apache.hadoop.hdfs.NameNodeProxies;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.ClientProtocol;
@@ -305,8 +304,7 @@ public class TestStripedINodeFile {
   public void testUnsuitableStoragePoliciesWithECStripedMode()
       throws Exception {
     final Configuration conf = new HdfsConfiguration();
-    int defaultStripedBlockSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE
-        * 4;
+    int defaultStripedBlockSize = testECPolicy.getCellSize() * 4;
     conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, defaultStripedBlockSize);
     conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
     conf.setLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 1L);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
index 6d4d797..6fd0a1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java
@@ -31,30 +31,33 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
 import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
 import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
 import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
 import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
 import org.apache.hadoop.hdfs.server.namenode.INodeFile;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
 import org.junit.Test;
 
 public class TestOfflineImageViewerWithStripedBlocks {
-  private static int dataBlocks = StripedFileTestUtil.NUM_DATA_BLOCKS;
-  private static int parityBlocks = StripedFileTestUtil.NUM_PARITY_BLOCKS;
+  private final ErasureCodingPolicy ecPolicy =
+      ErasureCodingPolicyManager.getSystemDefaultPolicy();
+  private int dataBlocks = ecPolicy.getNumDataUnits();
+  private int parityBlocks = ecPolicy.getNumParityUnits();
 
   private static MiniDFSCluster cluster;
   private static DistributedFileSystem fs;
-  private static final int cellSize = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-  private static final int stripesPerBlock = 3;
-  private static final int blockSize = cellSize * stripesPerBlock;
+  private final int cellSize = ecPolicy.getCellSize();
+  private final int stripesPerBlock = 3;
+  private final int blockSize = cellSize * stripesPerBlock;
 
-  @BeforeClass
-  public static void setup() throws IOException {
+  @Before
+  public void setup() throws IOException {
     int numDNs = dataBlocks + parityBlocks + 2;
     Configuration conf = new Configuration();
     conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize);
@@ -66,8 +69,8 @@ public class TestOfflineImageViewerWithStripedBlocks {
     fs.mkdirs(eczone);
   }
 
-  @AfterClass
-  public static void tearDown() {
+  @After
+  public void tearDown() {
     if (cluster != null) {
       cluster.shutdown();
     }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ffa116/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
index 999eb1f..c1728ac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.hdfs.util;
 import com.google.common.base.Preconditions;
 import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.DFSTestUtil;
-import org.apache.hadoop.hdfs.StripedFileTestUtil;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -81,15 +80,15 @@ import static org.junit.Assert.assertFalse;
  */
 public class TestStripedBlockUtil {
   // use hard coded policy - see HDFS-9816
-  private final ErasureCodingPolicy EC_POLICY =
+  private final ErasureCodingPolicy ecPolicy =
       ErasureCodingPolicyManager.getSystemPolicies()[0];
-  private final short DATA_BLK_NUM = (short) EC_POLICY.getNumDataUnits();
-  private final short PARITY_BLK_NUM = (short) EC_POLICY.getNumParityUnits();
-  private final short BLK_GROUP_WIDTH = (short) (DATA_BLK_NUM + PARITY_BLK_NUM);
-  private final int CELLSIZE = StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
-  private final int FULL_STRIPE_SIZE = DATA_BLK_NUM * CELLSIZE;
+  private final short dataBlocks = (short) ecPolicy.getNumDataUnits();
+  private final short parityBlocks = (short) ecPolicy.getNumParityUnits();
+  private final short groupSize = (short) (dataBlocks + parityBlocks);
+  private final int cellSize = ecPolicy.getCellSize();
+  private final int stripeSize = dataBlocks * cellSize;
   /** number of full stripes in a full block group */
-  private final int BLK_GROUP_STRIPE_NUM = 16;
+  private final int stripesPerBlock = 16;
   private final Random random = new Random();
 
   private int[] blockGroupSizes;
@@ -101,23 +100,23 @@ public class TestStripedBlockUtil {
 
   @Before
   public void setup(){
-    blockGroupSizes = new int[]{1, getDelta(CELLSIZE), CELLSIZE,
-        getDelta(DATA_BLK_NUM) * CELLSIZE,
-        getDelta(DATA_BLK_NUM) * CELLSIZE + getDelta(CELLSIZE),
-        FULL_STRIPE_SIZE, FULL_STRIPE_SIZE + getDelta(CELLSIZE),
-        FULL_STRIPE_SIZE + getDelta(DATA_BLK_NUM) * CELLSIZE,
-        FULL_STRIPE_SIZE + getDelta(DATA_BLK_NUM) * CELLSIZE + getDelta(CELLSIZE),
-        getDelta(BLK_GROUP_STRIPE_NUM) * FULL_STRIPE_SIZE,
-        BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE};
-    byteRangeStartOffsets = new int[] {0, getDelta(CELLSIZE), CELLSIZE - 1};
-    byteRangeSizes = new int[]{1, getDelta(CELLSIZE), CELLSIZE,
-        getDelta(DATA_BLK_NUM) * CELLSIZE,
-        getDelta(DATA_BLK_NUM) * CELLSIZE + getDelta(CELLSIZE),
-        FULL_STRIPE_SIZE, FULL_STRIPE_SIZE + getDelta(CELLSIZE),
-        FULL_STRIPE_SIZE + getDelta(DATA_BLK_NUM) * CELLSIZE,
-        FULL_STRIPE_SIZE + getDelta(DATA_BLK_NUM) * CELLSIZE + getDelta(CELLSIZE),
-        getDelta(BLK_GROUP_STRIPE_NUM) * FULL_STRIPE_SIZE,
-        BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE};
+    blockGroupSizes = new int[]{1, getDelta(cellSize), cellSize,
+        getDelta(dataBlocks) * cellSize,
+        getDelta(dataBlocks) * cellSize + getDelta(cellSize),
+        stripeSize, stripeSize + getDelta(cellSize),
+        stripeSize + getDelta(dataBlocks) * cellSize,
+        stripeSize + getDelta(dataBlocks) * cellSize + getDelta(cellSize),
+        getDelta(stripesPerBlock) * stripeSize,
+        stripesPerBlock * stripeSize};
+    byteRangeStartOffsets = new int[] {0, getDelta(cellSize), cellSize - 1};
+    byteRangeSizes = new int[]{1, getDelta(cellSize), cellSize,
+        getDelta(dataBlocks) * cellSize,
+        getDelta(dataBlocks) * cellSize + getDelta(cellSize),
+        stripeSize, stripeSize + getDelta(cellSize),
+        stripeSize + getDelta(dataBlocks) * cellSize,
+        stripeSize + getDelta(dataBlocks) * cellSize + getDelta(cellSize),
+        getDelta(stripesPerBlock) * stripeSize,
+        stripesPerBlock * stripeSize};
   }
 
   private int getDelta(int size) {
@@ -130,12 +129,12 @@ public class TestStripedBlockUtil {
 
   private LocatedStripedBlock createDummyLocatedBlock(int bgSize) {
     final long blockGroupID = -1048576;
-    DatanodeInfo[] locs = new DatanodeInfo[BLK_GROUP_WIDTH];
-    String[] storageIDs = new String[BLK_GROUP_WIDTH];
-    StorageType[] storageTypes = new StorageType[BLK_GROUP_WIDTH];
-    byte[] indices = new byte[BLK_GROUP_WIDTH];
-    for (int i = 0; i < BLK_GROUP_WIDTH; i++) {
-      indices[i] = (byte) ((i + 2) % DATA_BLK_NUM);
+    DatanodeInfo[] locs = new DatanodeInfo[groupSize];
+    String[] storageIDs = new String[groupSize];
+    StorageType[] storageTypes = new StorageType[groupSize];
+    byte[] indices = new byte[groupSize];
+    for (int i = 0; i < groupSize; i++) {
+      indices[i] = (byte) ((i + 2) % dataBlocks);
       // Location port always equal to logical index of a block,
       // for easier verification
       locs[i] = DFSTestUtil.getLocalDatanodeInfo(indices[i]);
@@ -148,20 +147,21 @@ public class TestStripedBlockUtil {
   }
 
   private byte[][] createInternalBlkBuffers(int bgSize) {
-    byte[][] bufs = new byte[DATA_BLK_NUM + PARITY_BLK_NUM][];
-    int[] pos = new int[DATA_BLK_NUM + PARITY_BLK_NUM];
-    for (int i = 0; i < DATA_BLK_NUM + PARITY_BLK_NUM; i++) {
+    byte[][] bufs = new byte[dataBlocks + parityBlocks][];
+    int[] pos = new int[dataBlocks + parityBlocks];
+    for (int i = 0; i < dataBlocks + parityBlocks; i++) {
       int bufSize = (int) getInternalBlockLength(
-          bgSize, CELLSIZE, DATA_BLK_NUM, i);
+          bgSize, cellSize, dataBlocks, i);
       bufs[i] = new byte[bufSize];
       pos[i] = 0;
     }
     int done = 0;
     while (done < bgSize) {
-      Preconditions.checkState(done % CELLSIZE == 0);
-      StripingCell cell = new StripingCell(EC_POLICY, CELLSIZE, done / CELLSIZE, 0);
+      Preconditions.checkState(done % cellSize == 0);
+      StripingCell cell =
+          new StripingCell(ecPolicy, cellSize, done / cellSize, 0);
       int idxInStripe = cell.idxInStripe;
-      int size = Math.min(CELLSIZE, bgSize - done);
+      int size = Math.min(cellSize, bgSize - done);
       for (int i = 0; i < size; i++) {
         bufs[idxInStripe][pos[idxInStripe] + i] = hashIntToByte(done + i);
       }
@@ -175,11 +175,11 @@ public class TestStripedBlockUtil {
   @Test
   public void testParseDummyStripedBlock() {
     LocatedStripedBlock lsb = createDummyLocatedBlock(
-        BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE);
+        stripeSize * stripesPerBlock);
     LocatedBlock[] blocks = parseStripedBlockGroup(
-        lsb, CELLSIZE, DATA_BLK_NUM, PARITY_BLK_NUM);
-    assertEquals(DATA_BLK_NUM + PARITY_BLK_NUM, blocks.length);
-    for (int i = 0; i < DATA_BLK_NUM; i++) {
+        lsb, cellSize, dataBlocks, parityBlocks);
+    assertEquals(dataBlocks + parityBlocks, blocks.length);
+    for (int i = 0; i < dataBlocks; i++) {
       assertFalse(blocks[i].isStriped());
       assertEquals(i,
           BlockIdManager.getBlockIndex(blocks[i].getBlock().getLocalBlock()));
@@ -191,9 +191,9 @@ public class TestStripedBlockUtil {
   }
 
   private void verifyInternalBlocks (int numBytesInGroup, int[] expected) {
-    for (int i = 1; i < BLK_GROUP_WIDTH; i++) {
+    for (int i = 1; i < groupSize; i++) {
       assertEquals(expected[i],
-          getInternalBlockLength(numBytesInGroup, CELLSIZE, DATA_BLK_NUM, i));
+          getInternalBlockLength(numBytesInGroup, cellSize, dataBlocks, i));
     }
   }
 
@@ -203,38 +203,38 @@ public class TestStripedBlockUtil {
     final int delta = 10;
 
     // Block group is smaller than a cell
-    verifyInternalBlocks(CELLSIZE - delta,
-        new int[] {CELLSIZE - delta, 0, 0, 0, 0, 0,
-            CELLSIZE - delta, CELLSIZE - delta, CELLSIZE - delta});
+    verifyInternalBlocks(cellSize - delta,
+        new int[] {cellSize - delta, 0, 0, 0, 0, 0,
+            cellSize - delta, cellSize - delta, cellSize - delta});
 
     // Block group is exactly as large as a cell
-    verifyInternalBlocks(CELLSIZE,
-        new int[] {CELLSIZE, 0, 0, 0, 0, 0,
-            CELLSIZE, CELLSIZE, CELLSIZE});
+    verifyInternalBlocks(cellSize,
+        new int[] {cellSize, 0, 0, 0, 0, 0,
+            cellSize, cellSize, cellSize});
 
     // Block group is a little larger than a cell
-    verifyInternalBlocks(CELLSIZE + delta,
-        new int[] {CELLSIZE, delta, 0, 0, 0, 0,
-            CELLSIZE, CELLSIZE, CELLSIZE});
+    verifyInternalBlocks(cellSize + delta,
+        new int[] {cellSize, delta, 0, 0, 0, 0,
+            cellSize, cellSize, cellSize});
 
     // Block group contains multiple stripes and ends at stripe boundary
-    verifyInternalBlocks(2 * DATA_BLK_NUM * CELLSIZE,
-        new int[] {2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
-            2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
-            2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE});
+    verifyInternalBlocks(2 * dataBlocks * cellSize,
+        new int[] {2 * cellSize, 2 * cellSize, 2 * cellSize,
+            2 * cellSize, 2 * cellSize, 2 * cellSize,
+            2 * cellSize, 2 * cellSize, 2 * cellSize});
 
     // Block group contains multiple stripes and ends at cell boundary
     // (not ending at stripe boundary)
-    verifyInternalBlocks(2 * DATA_BLK_NUM * CELLSIZE + CELLSIZE,
-        new int[] {3 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
-            2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
-            3 * CELLSIZE, 3 * CELLSIZE, 3 * CELLSIZE});
+    verifyInternalBlocks(2 * dataBlocks * cellSize + cellSize,
+        new int[] {3 * cellSize, 2 * cellSize, 2 * cellSize,
+            2 * cellSize, 2 * cellSize, 2 * cellSize,
+            3 * cellSize, 3 * cellSize, 3 * cellSize});
 
     // Block group contains multiple stripes and doesn't end at cell boundary
-    verifyInternalBlocks(2 * DATA_BLK_NUM * CELLSIZE - delta,
-        new int[] {2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE,
-            2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE - delta,
-            2 * CELLSIZE, 2 * CELLSIZE, 2 * CELLSIZE});
+    verifyInternalBlocks(2 * dataBlocks * cellSize - delta,
+        new int[] {2 * cellSize, 2 * cellSize, 2 * cellSize,
+            2 * cellSize, 2 * cellSize, 2 * cellSize - delta,
+            2 * cellSize, 2 * cellSize, 2 * cellSize});
   }
 
   /**
@@ -244,7 +244,7 @@ public class TestStripedBlockUtil {
   @Test
   public void testDivideByteRangeIntoStripes() {
     ByteBuffer assembled =
-        ByteBuffer.allocate(BLK_GROUP_STRIPE_NUM * FULL_STRIPE_SIZE);
+        ByteBuffer.allocate(stripesPerBlock * stripeSize);
     for (int bgSize : blockGroupSizes) {
       LocatedStripedBlock blockGroup = createDummyLocatedBlock(bgSize);
       byte[][] internalBlkBufs = createInternalBlkBuffers(bgSize);
@@ -253,11 +253,11 @@ public class TestStripedBlockUtil {
           if (brStart + brSize > bgSize) {
             continue;
           }
-          AlignedStripe[] stripes = divideByteRangeIntoStripes(EC_POLICY,
-              CELLSIZE, blockGroup, brStart, brStart + brSize - 1, assembled);
+          AlignedStripe[] stripes = divideByteRangeIntoStripes(ecPolicy,
+              cellSize, blockGroup, brStart, brStart + brSize - 1, assembled);
 
           for (AlignedStripe stripe : stripes) {
-            for (int i = 0; i < DATA_BLK_NUM; i++) {
+            for (int i = 0; i < dataBlocks; i++) {
               StripingChunk chunk = stripe.chunks[i];
               if (chunk == null || chunk.state != StripingChunk.REQUESTED) {
                 continue;


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message