Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 8EE4D181AD for ; Mon, 10 Aug 2015 16:48:19 +0000 (UTC) Received: (qmail 24697 invoked by uid 500); 10 Aug 2015 16:47:48 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 24493 invoked by uid 500); 10 Aug 2015 16:47:48 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 24192 invoked by uid 99); 10 Aug 2015 16:47:48 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 10 Aug 2015 16:47:48 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6C078E097A; Mon, 10 Aug 2015 16:47:48 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: aw@apache.org To: common-commits@hadoop.apache.org Date: Mon, 10 Aug 2015 16:48:01 -0000 Message-Id: In-Reply-To: References: X-Mailer: ASF-Git Admin Mailer Subject: [15/19] hadoop git commit: HDFS-8772. Fix TestStandbyIsHot#testDatanodeRestarts which occasionally fails. Contributed by Walter Su. HDFS-8772. Fix TestStandbyIsHot#testDatanodeRestarts which occasionally fails. Contributed by Walter Su. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98a27d11 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98a27d11 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98a27d11 Branch: refs/heads/HADOOP-12111 Commit: 98a27d110129c7b32455035831480f1c6197260b Parents: 4bc42d7 Author: Andrew Wang Authored: Wed Aug 5 16:35:41 2015 -0700 Committer: Andrew Wang Committed: Fri Aug 7 09:51:46 2015 -0700 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 30 +++++++++++++++++++- .../server/namenode/ha/TestStandbyIsHot.java | 2 ++ 3 files changed, 34 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/98a27d11/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 051dc8a..4e97b6b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -769,6 +769,9 @@ Release 2.8.0 - UNRELEASED HDFS-8856. Make LeaseManager#countPath O(1). (Arpit Agarwal) + HDFS-8772. Fix TestStandbyIsHot#testDatanodeRestarts which occasionally fails. + (Walter Su via wang) + OPTIMIZATIONS HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than http://git-wip-us.apache.org/repos/asf/hadoop/blob/98a27d11/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java index 0a21886..7052321 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java @@ -61,7 +61,9 @@ import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; +import java.util.concurrent.TimeoutException; +import com.google.common.base.Supplier; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; import org.apache.commons.logging.Log; @@ -86,6 +88,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock; import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType; import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil; import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor; +import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Util; @@ -114,6 +117,7 @@ import org.apache.hadoop.net.StaticMapping; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.security.authorize.ProxyUsers; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; @@ -2386,7 +2390,31 @@ public class MiniDFSCluster { client.close(); } - + + /** Wait until the given namenode gets first block reports from all the datanodes */ + public void waitFirstBRCompleted(int nnIndex, int timeout) throws + IOException, TimeoutException, InterruptedException { + if (namenodes.size() == 0 || getNN(nnIndex) == null || getNN(nnIndex).nameNode == null) { + return; + } + + final FSNamesystem ns = getNamesystem(nnIndex); + final DatanodeManager dm = ns.getBlockManager().getDatanodeManager(); + GenericTestUtils.waitFor(new Supplier() { + @Override + public Boolean get() { + List nodes = dm.getDatanodeListForReport + (DatanodeReportType.LIVE); + for (DatanodeDescriptor node : nodes) { + if (!node.checkBlockReportReceived()) { + return false; + } + } + return true; + } + }, 100, timeout); + } + /** * Wait until the cluster is active and running. */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/98a27d11/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java index 622ed94..14c9dc2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestStandbyIsHot.java @@ -182,6 +182,8 @@ public class TestStandbyIsHot { // Wait for both NNs to re-register the DN. cluster.waitActive(0); cluster.waitActive(1); + cluster.waitFirstBRCompleted(0, 10000); + cluster.waitFirstBRCompleted(1, 10000); BlockManagerTestUtil.updateState(nn0.getNamesystem().getBlockManager()); BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());