Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 7D952183A8 for ; Wed, 20 Jan 2016 07:21:36 +0000 (UTC) Received: (qmail 26116 invoked by uid 500); 20 Jan 2016 07:21:36 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 26052 invoked by uid 500); 20 Jan 2016 07:21:36 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 26043 invoked by uid 99); 20 Jan 2016 07:21:36 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 20 Jan 2016 07:21:36 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 0628FDFF93; Wed, 20 Jan 2016 07:21:35 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: yjzhangal@apache.org To: common-commits@hadoop.apache.org Message-Id: X-Mailer: ASF-Git Admin Mailer Subject: hadoop git commit: HDFS-6054. MiniQJMHACluster should not use static port to avoid binding failure in unit test. (Yongjun Zhang) Date: Wed, 20 Jan 2016 07:21:35 +0000 (UTC) Repository: hadoop Updated Branches: refs/heads/branch-2.8 a359dc87d -> 15713f947 HDFS-6054. MiniQJMHACluster should not use static port to avoid binding failure in unit test. (Yongjun Zhang) (cherry picked from commit 57d0a94305b22a469062c7ce85b3159fee17a28b) Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java (cherry picked from commit a5ee4c09de849b1fdb1ee69b2e40a071a7050456) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15713f94 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15713f94 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15713f94 Branch: refs/heads/branch-2.8 Commit: 15713f947cf1b936957446bd6d9a6962e880fb15 Parents: a359dc8 Author: Yongjun Zhang Authored: Tue Jan 19 22:54:47 2016 -0800 Committer: Yongjun Zhang Committed: Tue Jan 19 23:14:56 2016 -0800 ---------------------------------------------------------------------- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 17 +++++--- .../namenode/ha/TestFailureToReadEdits.java | 41 ++++++++++++++++---- 3 files changed, 48 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/15713f94/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index da4bbe2..04a323f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1663,6 +1663,9 @@ Release 2.8.0 - UNRELEASED HDFS-9623. Update example configuration of block state change log in log4j.properties. (Masatake Iwasaki via aajisaka) + HDFS-6054. MiniQJMHACluster should not use static port to avoid binding + failure in unit test. (Yongjun Zhang) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/15713f94/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java index c642a07..5caccf4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/MiniQJMHACluster.java @@ -47,7 +47,6 @@ public class MiniQJMHACluster { private static final String NN1 = "nn1"; private static final String NN2 = "nn2"; private static final Random RANDOM = new Random(); - private int basePort = 10000; public static class Builder { private final Configuration conf; @@ -86,9 +85,12 @@ public class MiniQJMHACluster { private MiniQJMHACluster(Builder builder) throws IOException { this.conf = builder.conf; int retryCount = 0; + int basePort = 10000; + while (true) { try { basePort = 10000 + RANDOM.nextInt(1000) * 4; + LOG.info("Set MiniQJMHACluster basePort to " + basePort); // start 3 journal nodes journalCluster = new MiniJournalCluster.Builder(conf).format(true) .build(); @@ -98,7 +100,7 @@ public class MiniQJMHACluster { // start cluster with 2 NameNodes MiniDFSNNTopology topology = createDefaultTopology(basePort); - initHAConf(journalURI, builder.conf); + initHAConf(journalURI, builder.conf, basePort); // First start up the NNs just to format the namespace. The MinIDFSCluster // has no way to just format the NameNodes without also starting them. @@ -116,16 +118,21 @@ public class MiniQJMHACluster { // restart the cluster cluster.restartNameNodes(); - ++retryCount; break; } catch (BindException e) { + if (cluster != null) { + cluster.shutdown(true); + cluster = null; + } + ++retryCount; LOG.info("MiniQJMHACluster port conflicts, retried " + retryCount + " times"); } } } - - private Configuration initHAConf(URI journalURI, Configuration conf) { + + private Configuration initHAConf(URI journalURI, Configuration conf, + int basePort) { conf.set(DFSConfigKeys.DFS_NAMENODE_SHARED_EDITS_DIR_KEY, journalURI.toString()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/15713f94/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java index f82b616..5221ef9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestFailureToReadEdits.java @@ -27,10 +27,14 @@ import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.spy; import java.io.IOException; +import java.net.BindException; import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; +import java.util.Random; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -62,11 +66,14 @@ import com.google.common.collect.ImmutableList; @RunWith(Parameterized.class) public class TestFailureToReadEdits { + private static final Log LOG = + LogFactory.getLog(TestFailureToReadEdits.class); private static final String TEST_DIR1 = "/test1"; private static final String TEST_DIR2 = "/test2"; private static final String TEST_DIR3 = "/test3"; - + private static final Random RANDOM = new Random(); + private final TestType clusterType; private Configuration conf; private MiniDFSCluster cluster; @@ -103,14 +110,32 @@ public class TestFailureToReadEdits { conf.setInt(DFSConfigKeys.DFS_NAMENODE_NUM_CHECKPOINTS_RETAINED_KEY, 10); conf.setInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY, 1); HAUtil.setAllowStandbyReads(conf, true); - + if (clusterType == TestType.SHARED_DIR_HA) { - MiniDFSNNTopology topology = MiniQJMHACluster.createDefaultTopology(10000); - cluster = new MiniDFSCluster.Builder(conf) - .nnTopology(topology) - .numDataNodes(0) - .checkExitOnShutdown(false) - .build(); + int basePort = 10000; + int retryCount = 0; + while (true) { + try { + basePort = 10000 + RANDOM.nextInt(1000) * 4; + LOG.info("Set SHARED_DIR_HA cluster's basePort to " + basePort); + MiniDFSNNTopology topology = + MiniQJMHACluster.createDefaultTopology(basePort); + cluster = new MiniDFSCluster.Builder(conf) + .nnTopology(topology) + .numDataNodes(0) + .checkExitOnShutdown(false) + .build(); + break; + } catch (BindException e) { + if (cluster != null) { + cluster.shutdown(true); + cluster = null; + } + ++retryCount; + LOG.info("SHARED_DIR_HA: MiniQJMHACluster port conflicts, retried " + + retryCount + " times " + e); + } + } } else { Builder builder = new MiniQJMHACluster.Builder(conf); builder.getDfsBuilder().numDataNodes(0).checkExitOnShutdown(false);