This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 14282e3 HDFS-14188. Make hdfs ec -verifyClusterSetup command accept an erasure coding
policy as a parameter. Contributed by Kitti Nanasi.
14282e3 is described below
commit 14282e311be6ffcaddd2f74fa8e67c4e98a32291
Author: Kitti Nanasi <knanasi@cloudera.com>
AuthorDate: Tue Feb 19 12:04:56 2019 -0800
HDFS-14188. Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as
a parameter. Contributed by Kitti Nanasi.
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
---
.../java/org/apache/hadoop/hdfs/tools/ECAdmin.java | 40 ++++++++++++---
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +
.../src/site/markdown/HDFSErasureCoding.md | 5 ++
.../org/apache/hadoop/hdfs/tools/TestECAdmin.java | 58 +++++++++++++++++++++-
4 files changed, 95 insertions(+), 10 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 6dccccb..80d1b6f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.shell.CommandFormat;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -612,24 +613,32 @@ public class ECAdmin extends Configured implements Tool {
@Override
public String getShortUsage() {
- return "[" + getName() + "]\n";
+ return "[" + getName() + " [-policy <policy>...<policy>]]\n";
}
@Override
public String getLongUsage() {
+ TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<policy>", "The name of the erasure coding policy");
return getShortUsage() + "\n"
- + "Verify the cluster setup can support all enabled erasure coding"
- + " policies.\n";
+ + "Verify if the cluster setup can support all enabled erasure " +
+ "coding policies. If optional parameter -policy is specified, " +
+ "verify if the cluster setup can support the given policy.\n";
}
@Override
public int run(Configuration conf, List<String> args) throws IOException {
- if (args.size() > 0) {
- System.err.println(getName() + ": Too many arguments");
- return 1;
- }
+ boolean isPolicyOption = StringUtils.popOption("-policy", args);
final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
- ECTopologyVerifierResult result = getECTopologyVerifierResult(dfs);
+ ECTopologyVerifierResult result;
+ if (isPolicyOption) {
+ CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
+ c.parse(args);
+ String[] parameters = args.toArray(new String[args.size()]);
+ result = getECTopologyResultForPolicies(dfs, parameters);
+ } else {
+ result = getECTopologyVerifierResult(dfs);
+ }
System.out.println(result.getResultMessage());
if (result.isSupported()) {
return 0;
@@ -649,6 +658,21 @@ public class ECAdmin extends Configured implements Tool {
getEnabledPolicies(policies));
}
+ private static ECTopologyVerifierResult getECTopologyResultForPolicies(
+ final DistributedFileSystem dfs, final String... policyNames)
+ throws IOException {
+ ErasureCodingPolicy[] policies =
+ new ErasureCodingPolicy[policyNames.length];
+ for (int i = 0; i < policyNames.length; i++) {
+ policies[i] =
+ getPolicy(dfs.getClient().getNamenode().getErasureCodingPolicies(),
+ policyNames[i]);
+ }
+ final DatanodeInfo[] report = dfs.getClient().getNamenode()
+ .getDatanodeReport(HdfsConstants.DatanodeReportType.ALL);
+ return ECTopologyVerifier.getECTopologyVerifierResult(report, policies);
+ }
+
private static ECTopologyVerifierResult getECTopologyVerifierResultForPolicy(
final DistributedFileSystem dfs, final String policyName)
throws IOException {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index c0443f7..0ba9b94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -488,6 +488,7 @@ Usage:
[-listCodecs]
[-enablePolicy -policy <policyName>]
[-disablePolicy -policy <policyName>]
+ [-verifyClusterSetup -policy <policyName>...<policyName>]
[-help [cmd ...]]
| COMMAND\_OPTION | Description |
@@ -500,6 +501,7 @@ Usage:
|-listCodecs| Get the list of supported erasure coding codecs and coders in system|
|-enablePolicy| Enable an ErasureCoding policy in system|
|-disablePolicy| Disable an ErasureCoding policy in system|
+|-verifyClusterSetup| Verify if the cluster setup can support a list of erasure coding policies|
Runs the ErasureCoding CLI. See [HDFS ErasureCoding](./HDFSErasureCoding.html#Administrative_commands)
for more information on this command.
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 67e6b75..3a8b611 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -170,6 +170,7 @@ Deployment
[-listCodecs]
[-enablePolicy -policy <policyName>]
[-disablePolicy -policy <policyName>]
+ [-verifyClusterSetup -policy <policyName>...<policyName>]
[-help [cmd ...]]
Below are the details about each command.
@@ -221,6 +222,10 @@ Below are the details about each command.
Disable an erasure coding policy.
+* `[-verifyClusterSetup -policy <policyName>...<policyName>]`
+
+ Verify if the cluster setup can support all enabled erasure coding policies. If optional
parameter -policy is specified, verify if the cluster setup can support the given policy or
policies.
+
Limitations
-----------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java
index 21a574c..410298f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestECAdmin.java
@@ -68,8 +68,7 @@ public class TestECAdmin {
try {
System.out.flush();
System.err.flush();
- out.reset();
- err.reset();
+ resetOutputs();
} finally {
System.setOut(OLD_OUT);
System.setErr(OLD_ERR);
@@ -242,4 +241,59 @@ public class TestECAdmin {
err.toString().contains("RemoteException: The policy name " +
"NonExistentPolicy does not exist"));
}
+
+ @Test
+ public void testVerifyClusterSetupWithGivenPolicies() throws Exception {
+ cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0);
+
+ String[] args = new String[]{"-verifyClusterSetup", "-policy",
+ "RS-3-2-1024k"};
+ int ret = admin.run(args);
+ LOG.info("Command stdout: {}", out.toString());
+ LOG.info("Command stderr: {}", err.toString());
+ assertEquals("Return value of the command is not successful", 2, ret);
+ assertTrue("Result of cluster topology verify " +
+ "should be logged correctly", out.toString()
+ .contains("less than the minimum required number of racks (3) " +
+ "for the erasure coding policies: RS-3-2-1024k"));
+ assertTrue("Error output should be empty", err.toString().isEmpty());
+
+ resetOutputs();
+ args = new String[]{"-verifyClusterSetup", "-policy",
+ "RS-10-4-1024k", "RS-3-2-1024k"};
+ ret = admin.run(args);
+ LOG.info("Command stdout: {}", out.toString());
+ LOG.info("Command stderr: {}", err.toString());
+ assertEquals("Return value of the command is not successful", 2, ret);
+ assertTrue("Result of cluster topology verify " +
+ "should be logged correctly", out.toString()
+ .contains(
+ "for the erasure coding policies: RS-10-4-1024k, RS-3-2-1024k"));
+ assertTrue("Error output should be empty", err.toString().isEmpty());
+
+ resetOutputs();
+ args = new String[]{"-verifyClusterSetup", "-policy", "invalidPolicy"};
+ ret = admin.run(args);
+ LOG.info("Command stdout: {}", out.toString());
+ LOG.info("Command stderr: {}", err.toString());
+ assertEquals("Return value of the command is not successful", -1, ret);
+ assertTrue("Error message should be logged", err.toString()
+ .contains("The given erasure coding policy invalidPolicy " +
+ "does not exist."));
+
+ resetOutputs();
+ args = new String[]{"-verifyClusterSetup", "-policy"};
+ ret = admin.run(args);
+ LOG.info("Command stdout: {}", out.toString());
+ LOG.info("Command stderr: {}", err.toString());
+ assertEquals("Return value of the command is not successful", -1, ret);
+ assertTrue("Error message should be logged", err.toString()
+ .contains("NotEnoughArgumentsException: Not enough arguments: " +
+ "expected 1 but got 0"));
+ }
+
+ private void resetOutputs() {
+ out.reset();
+ err.reset();
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
|