From common-commits-return-79896-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Fri Mar 16 18:32:01 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 1F87A18077F for ; Fri, 16 Mar 2018 18:32:00 +0100 (CET) Received: (qmail 29223 invoked by uid 500); 16 Mar 2018 17:31:59 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 28966 invoked by uid 99); 16 Mar 2018 17:31:59 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 16 Mar 2018 17:31:59 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id D8ACBF1802; Fri, 16 Mar 2018 17:31:58 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: inigoiri@apache.org To: common-commits@hadoop.apache.org Date: Fri, 16 Mar 2018 17:32:02 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [5/5] hadoop git commit: HDFS-12886. Ignore minReplication for block recovery. Contributed by Lukas Majercak. HDFS-12886. Ignore minReplication for block recovery. Contributed by Lukas Majercak. (cherry picked from commit 08ff1586d5d3e39f546200f9e696f62ea4cf000d) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0f7050b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0f7050b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0f7050b Branch: refs/heads/branch-2.9 Commit: d0f7050b9cbcba3d90d5a1fc7d46cac9ddc909e4 Parents: 6934145 Author: Inigo Goiri Authored: Fri Mar 16 10:29:19 2018 -0700 Committer: Inigo Goiri Committed: Fri Mar 16 10:31:28 2018 -0700 ---------------------------------------------------------------------- .../server/blockmanagement/BlockManager.java | 7 ++ .../hdfs/server/datanode/TestBlockRecovery.java | 83 ++++++++++++++++++++ 2 files changed, 90 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0f7050b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java index 4a8c8c2..9319633 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java @@ -847,6 +847,13 @@ public class BlockManager implements BlockStatsMXBean { addExpectedReplicasToPending(lastBlock); } completeBlock(lastBlock, iip, false); + } else if (pendingRecoveryBlocks.isUnderRecovery(lastBlock)) { + // We've just finished recovery for this block, complete + // the block forcibly disregarding number of replicas. + // This is to ignore minReplication, the block will be closed + // and then replicated out. + completeBlock(lastBlock, iip, true); + updateNeededReconstructions(lastBlock, 1, 0); } return committed; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0f7050b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java index 1aa538d..15a416d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java @@ -19,9 +19,14 @@ package org.apache.hadoop.hdfs.server.datanode; import org.apache.hadoop.hdfs.AppendTestUtil; +import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.server.namenode.NameNode; import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BLOCK_SIZE_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY; +import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REPLICATION_MIN_KEY; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Matchers.any; @@ -42,6 +47,7 @@ import java.io.IOException; import java.net.InetSocketAddress; import java.net.URISyntaxException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.List; import java.util.Random; @@ -1087,4 +1093,81 @@ public class TestBlockRecovery { } } } + + /** + * Test that block will be recovered even if there are less than the + * specified minReplication datanodes involved in its recovery. + * + * Check that, after recovering, the block will be successfully replicated. + */ + @Test(timeout = 300000L) + public void testRecoveryWillIgnoreMinReplication() throws Exception { + tearDown(); // Stop the Mocked DN started in startup() + + final int blockSize = 4096; + final int numReplicas = 3; + final String filename = "/testIgnoreMinReplication"; + final Path filePath = new Path(filename); + Configuration configuration = new HdfsConfiguration(); + configuration.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 2000); + configuration.setInt(DFS_NAMENODE_REPLICATION_MIN_KEY, 2); + configuration.setLong(DFS_BLOCK_SIZE_KEY, blockSize); + MiniDFSCluster cluster = null; + + try { + cluster = new MiniDFSCluster.Builder(configuration).numDataNodes(5) + .build(); + cluster.waitActive(); + final DistributedFileSystem dfs = cluster.getFileSystem(); + final FSNamesystem fsn = cluster.getNamesystem(); + + // Create a file and never close the output stream to trigger recovery + FSDataOutputStream out = dfs.create(filePath, (short) numReplicas); + out.write(AppendTestUtil.randomBytes(0, blockSize)); + out.hsync(); + + DFSClient dfsClient = new DFSClient(new InetSocketAddress("localhost", + cluster.getNameNodePort()), configuration); + LocatedBlock blk = dfsClient.getNamenode(). + getBlockLocations(filename, 0, blockSize). + getLastLocatedBlock(); + + // Kill 2 out of 3 datanodes so that only 1 alive, thus < minReplication + List dataNodes = Arrays.asList(blk.getLocations()); + assertEquals(dataNodes.size(), numReplicas); + for (DatanodeInfo dataNode : dataNodes.subList(0, numReplicas - 1)) { + cluster.stopDataNode(dataNode.getName()); + } + + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + return fsn.getNumDeadDataNodes() == 2; + } + }, 300, 300000); + + // Make sure hard lease expires to trigger replica recovery + cluster.setLeasePeriod(100L, 100L); + + // Wait for recovery to succeed + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + try { + return dfs.isFileClosed(filePath); + } catch (IOException e) {} + return false; + } + }, 300, 300000); + + // Wait for the block to be replicated + DFSTestUtil.waitForReplication(cluster, DFSTestUtil.getFirstBlock( + dfs, filePath), 1, numReplicas, 0); + + } finally { + if (cluster != null) { + cluster.shutdown(); + } + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org