hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dran...@apache.org
Subject [29/32] hadoop git commit: HDFS-10858. FBR processing may generate incorrect reportedBlock-blockGroup mapping. Contributed by Jing Zhao.
Date Tue, 13 Sep 2016 08:16:53 GMT
HDFS-10858. FBR processing may generate incorrect reportedBlock-blockGroup mapping. Contributed
by Jing Zhao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72dfb048
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72dfb048
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72dfb048

Branch: refs/heads/HADOOP-12756
Commit: 72dfb048a9a7be64b371b74478b90150bf300d35
Parents: 59d5966
Author: Jing Zhao <jing9@apache.org>
Authored: Mon Sep 12 16:38:39 2016 -0700
Committer: Jing Zhao <jing9@apache.org>
Committed: Mon Sep 12 16:40:11 2016 -0700

----------------------------------------------------------------------
 .../server/blockmanagement/BlockManager.java    |   4 +-
 .../namenode/TestAddStripedBlockInFBR.java      | 109 +++++++++++++++++++
 2 files changed, 111 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/72dfb048/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 1362c0b..3a12d74 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -2624,7 +2624,7 @@ public class BlockManager implements BlockStatsMXBean {
       } while (storageBlock != null);
     }
 
-    // Iterate any remaing blocks that have not been reported and remove them
+    // Iterate any remaining blocks that have not been reported and remove them
     while (storageBlocksIterator.hasNext()) {
       toRemove.add(storageBlocksIterator.next());
     }
@@ -2677,7 +2677,7 @@ public class BlockManager implements BlockStatsMXBean {
                 corruptReplicas.isReplicaCorrupt(storedBlock, dn))) {
       // Add replica if appropriate. If the replica was previously corrupt
       // but now okay, it might need to be updated.
-      toAdd.add(new BlockInfoToAdd(storedBlock, replica));
+      toAdd.add(new BlockInfoToAdd(storedBlock, new Block(replica)));
     }
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/72dfb048/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
new file mode 100644
index 0000000..37b334f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.BLOCK_STRIPED_CELL_SIZE;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_DATA_BLOCKS;
+import static org.apache.hadoop.hdfs.StripedFileTestUtil.NUM_PARITY_BLOCKS;
+
+public class TestAddStripedBlockInFBR {
+  private final short GROUP_SIZE = (short) (NUM_DATA_BLOCKS + NUM_PARITY_BLOCKS);
+
+  private MiniDFSCluster cluster;
+  private DistributedFileSystem dfs;
+
+  @Rule
+  public Timeout globalTimeout = new Timeout(300000);
+
+  @Before
+  public void setup() throws IOException {
+    Configuration conf = new HdfsConfiguration();
+    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(GROUP_SIZE).build();
+    cluster.waitActive();
+    dfs = cluster.getFileSystem();
+  }
+
+  @After
+  public void tearDown() {
+    if (cluster != null) {
+      cluster.shutdown();
+      cluster = null;
+    }
+  }
+
+  @Test
+  public void testAddBlockInFullBlockReport() throws Exception {
+    BlockManager spy = Mockito.spy(cluster.getNamesystem().getBlockManager());
+    // let NN ignore one DataNode's IBR
+    final DataNode dn = cluster.getDataNodes().get(0);
+    final DatanodeID datanodeID = dn.getDatanodeId();
+    Mockito.doNothing().when(spy)
+        .processIncrementalBlockReport(Mockito.eq(datanodeID), Mockito.any());
+    Whitebox.setInternalState(cluster.getNamesystem(), "blockManager", spy);
+
+    final Path ecDir = new Path("/ec");
+    final Path repDir = new Path("/rep");
+    dfs.mkdirs(ecDir);
+    dfs.mkdirs(repDir);
+    dfs.getClient().setErasureCodingPolicy(ecDir.toString(), null);
+
+    // create several non-EC files and one EC file
+    final Path[] repFiles = new Path[GROUP_SIZE];
+    for (int i = 0; i < GROUP_SIZE; i++) {
+      repFiles[i] = new Path(repDir, "f" + i);
+      DFSTestUtil.createFile(dfs, repFiles[i], 1L, (short) 3, 0L);
+    }
+    final Path ecFile = new Path(ecDir, "f");
+    DFSTestUtil.createFile(dfs, ecFile,
+        BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS, (short) 1, 0L);
+
+    // trigger dn's FBR. The FBR will add block-dn mapping.
+    DataNodeTestUtils.triggerBlockReport(dn);
+
+    // make sure NN has correct block-dn mapping
+    BlockInfoStriped blockInfo = (BlockInfoStriped) cluster.getNamesystem()
+        .getFSDirectory().getINode(ecFile.toString()).asFile().getLastBlock();
+    NumberReplicas nr = spy.countNodes(blockInfo);
+    Assert.assertEquals(GROUP_SIZE, nr.liveReplicas());
+    Assert.assertEquals(0, nr.excessReplicas());
+  }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message