From common-commits-return-93214-archive-asf-public=cust-asf.ponee.io@hadoop.apache.org Tue Feb 19 20:05:30 2019 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 2ADDD18060E for ; Tue, 19 Feb 2019 21:05:29 +0100 (CET) Received: (qmail 61584 invoked by uid 500); 19 Feb 2019 20:05:29 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 61575 invoked by uid 99); 19 Feb 2019 20:05:29 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Tue, 19 Feb 2019 20:05:29 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 8927085180; Tue, 19 Feb 2019 20:05:28 +0000 (UTC) Date: Tue, 19 Feb 2019 20:05:28 +0000 To: "common-commits@hadoop.apache.org" Subject: [hadoop] branch trunk updated: HDFS-14188. Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as a parameter. Contributed by Kitti Nanasi. MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <155060672803.20568.4294671361632748466@gitbox.apache.org> From: weichiu@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: hadoop X-Git-Refname: refs/heads/trunk X-Git-Reftype: branch X-Git-Oldrev: b8de78c570babe4f802d951957c495ea0a4b07da X-Git-Newrev: 14282e311be6ffcaddd2f74fa8e67c4e98a32291 X-Git-Rev: 14282e311be6ffcaddd2f74fa8e67c4e98a32291 X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. weichiu pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/hadoop.git The following commit(s) were added to refs/heads/trunk by this push: new 14282e3 HDFS-14188. Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as a parameter. Contributed by Kitti Nanasi. 14282e3 is described below commit 14282e311be6ffcaddd2f74fa8e67c4e98a32291 Author: Kitti Nanasi AuthorDate: Tue Feb 19 12:04:56 2019 -0800 HDFS-14188. Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as a parameter. Contributed by Kitti Nanasi. Signed-off-by: Wei-Chiu Chuang --- .../java/org/apache/hadoop/hdfs/tools/ECAdmin.java | 40 ++++++++++++--- .../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 + .../src/site/markdown/HDFSErasureCoding.md | 5 ++ .../org/apache/hadoop/hdfs/tools/TestECAdmin.java | 58 +++++++++++++++++++++- 4 files changed, 95 insertions(+), 10 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 6dccccb..80d1b6f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; +import org.apache.hadoop.fs.shell.CommandFormat; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; @@ -612,24 +613,32 @@ public class ECAdmin extends Configured implements Tool { @Override public String getShortUsage() { - return "[" + getName() + "]\n"; + return "[" + getName() + " [-policy ...]]\n"; } @Override public String getLongUsage() { + TableListing listing = AdminHelper.getOptionDescriptionListing(); + listing.addRow("", "The name of the erasure coding policy"); return getShortUsage() + "\n" - + "Verify the cluster setup can support all enabled erasure coding" - + " policies.\n"; + + "Verify if the cluster setup can support all enabled erasure " + + "coding policies. If optional parameter -policy is specified, " + + "verify if the cluster setup can support the given policy.\n"; } @Override public int run(Configuration conf, List args) throws IOException { - if (args.size() > 0) { - System.err.println(getName() + ": Too many arguments"); - return 1; - } + boolean isPolicyOption = StringUtils.popOption("-policy", args); final DistributedFileSystem dfs = AdminHelper.getDFS(conf); - ECTopologyVerifierResult result = getECTopologyVerifierResult(dfs); + ECTopologyVerifierResult result; + if (isPolicyOption) { + CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE); + c.parse(args); + String[] parameters = args.toArray(new String[args.size()]); + result = getECTopologyResultForPolicies(dfs, parameters); + } else { + result = getECTopologyVerifierResult(dfs); + } System.out.println(result.getResultMessage()); if (result.isSupported()) { return 0; @@ -649,6 +658,21 @@ public class ECAdmin extends Configured implements Tool { getEnabledPolicies(policies)); } + private static ECTopologyVerifierResult getECTopologyResultForPolicies( + final DistributedFileSystem dfs, final String... policyNames) + throws IOException { + ErasureCodingPolicy[] policies = + new ErasureCodingPolicy[policyNames.length]; + for (int i = 0; i < policyNames.length; i++) { + policies[i] = + getPolicy(dfs.getClient().getNamenode().getErasureCodingPolicies(), + policyNames[i]); + } + final DatanodeInfo[] report = dfs.getClient().getNamenode() + .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL); + return ECTopologyVerifier.getECTopologyVerifierResult(report, policies); + } + private static ECTopologyVerifierResult getECTopologyVerifierResultForPolicy( final DistributedFileSystem dfs, final String policyName) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index c0443f7..0ba9b94 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -488,6 +488,7 @@ Usage: [-listCodecs] [-enablePolicy -policy ] [-disablePolicy -policy ] + [-verifyClusterSetup -policy ...] [-help [cmd ...]] | COMMAND\_OPTION | Description | @@ -500,6 +501,7 @@ Usage: |-listCodecs| Get the list of supported erasure coding codecs and coders in system| |-enablePolicy| Enable an ErasureCoding policy in system| |-disablePolicy| Disable an ErasureCoding policy in system| +|-verifyClusterSetup| Verify if the cluster setup can support a list of erasure coding policies| Runs the ErasureCoding CLI. See [HDFS ErasureCoding](./HDFSErasureCoding.html#Administrative_commands) for more information on this command. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md index 67e6b75..3a8b611 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md @@ -170,6 +170,7 @@ Deployment [-listCodecs] [-enablePolicy -policy ] [-disablePolicy -policy ] + [-verifyClusterSetup -policy ...] [-help [cmd ...]] Below are the details about each command. @@ -221,6 +222,10 @@ Below are the details about each command. Disable an erasure coding policy. +* `[-verifyClusterSetup -policy ...]` + + Verify if the cluster setup can support all enabled erasure coding policies. If optional parameter -policy is specified, verify if the cluster setup can support the given policy or policies. + Limitations ----------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java index 21a574c..410298f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java @@ -68,8 +68,7 @@ public class TestECAdmin { try { System.out.flush(); System.err.flush(); - out.reset(); - err.reset(); + resetOutputs(); } finally { System.setOut(OLD_OUT); System.setErr(OLD_ERR); @@ -242,4 +241,59 @@ public class TestECAdmin { err.toString().contains("RemoteException: The policy name " + "NonExistentPolicy does not exist")); } + + @Test + public void testVerifyClusterSetupWithGivenPolicies() throws Exception { + cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0); + + String[] args = new String[]{"-verifyClusterSetup", "-policy", + "RS-3-2-1024k"}; + int ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", 2, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString() + .contains("less than the minimum required number of racks (3) " + + "for the erasure coding policies: RS-3-2-1024k")); + assertTrue("Error output should be empty", err.toString().isEmpty()); + + resetOutputs(); + args = new String[]{"-verifyClusterSetup", "-policy", + "RS-10-4-1024k", "RS-3-2-1024k"}; + ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", 2, ret); + assertTrue("Result of cluster topology verify " + + "should be logged correctly", out.toString() + .contains( + "for the erasure coding policies: RS-10-4-1024k, RS-3-2-1024k")); + assertTrue("Error output should be empty", err.toString().isEmpty()); + + resetOutputs(); + args = new String[]{"-verifyClusterSetup", "-policy", "invalidPolicy"}; + ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", -1, ret); + assertTrue("Error message should be logged", err.toString() + .contains("The given erasure coding policy invalidPolicy " + + "does not exist.")); + + resetOutputs(); + args = new String[]{"-verifyClusterSetup", "-policy"}; + ret = admin.run(args); + LOG.info("Command stdout: {}", out.toString()); + LOG.info("Command stderr: {}", err.toString()); + assertEquals("Return value of the command is not successful", -1, ret); + assertTrue("Error message should be logged", err.toString() + .contains("NotEnoughArgumentsException: Not enough arguments: " + + "expected 1 but got 0")); + } + + private void resetOutputs() { + out.reset(); + err.reset(); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org