Return-Path: X-Original-To: archive-asf-public-internal@cust-asf2.ponee.io Delivered-To: archive-asf-public-internal@cust-asf2.ponee.io Received: from cust-asf.ponee.io (cust-asf.ponee.io [163.172.22.183]) by cust-asf2.ponee.io (Postfix) with ESMTP id 3513C200C4F for ; Fri, 3 Mar 2017 03:45:27 +0100 (CET) Received: by cust-asf.ponee.io (Postfix) id 31CAC160B7D; Fri, 3 Mar 2017 02:45:27 +0000 (UTC) Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by cust-asf.ponee.io (Postfix) with SMTP id 236FD160B7A for ; Fri, 3 Mar 2017 03:45:24 +0100 (CET) Received: (qmail 40121 invoked by uid 500); 3 Mar 2017 02:45:14 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 38712 invoked by uid 99); 3 Mar 2017 02:45:13 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 03 Mar 2017 02:45:13 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 6716EF3206; Fri, 3 Mar 2017 02:45:13 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: jhung@apache.org To: common-commits@hadoop.apache.org Date: Fri, 03 Mar 2017 02:45:48 -0000 Message-Id: In-Reply-To: <664e5264334847449ff22ddccac06f6f@git.apache.org> References: <664e5264334847449ff22ddccac06f6f@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [37/50] [abbrv] hadoop git commit: HDFS-11428. Change setErasureCodingPolicy to take a required string EC policy name. Contributed by Andrew Wang. archived-at: Fri, 03 Mar 2017 02:45:27 -0000 HDFS-11428. Change setErasureCodingPolicy to take a required string EC policy name. Contributed by Andrew Wang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82ef9acc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82ef9acc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82ef9acc Branch: refs/heads/YARN-5734 Commit: 82ef9accafe7318278efb169678e17065e082c8e Parents: dcd03df Author: Rakesh Radhakrishnan Authored: Wed Mar 1 16:06:02 2017 +0530 Committer: Rakesh Radhakrishnan Committed: Wed Mar 1 16:06:02 2017 +0530 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hdfs/DFSClient.java | 4 +-- .../hadoop/hdfs/DistributedFileSystem.java | 11 +++--- .../apache/hadoop/hdfs/client/HdfsAdmin.java | 10 +++--- .../hadoop/hdfs/protocol/ClientProtocol.java | 5 ++- .../ClientNamenodeProtocolTranslatorPB.java | 6 ++-- .../src/main/proto/erasurecoding.proto | 2 +- ...tNamenodeProtocolServerSideTranslatorPB.java | 4 +-- .../server/namenode/FSDirErasureCodingOp.java | 13 +++++-- .../hdfs/server/namenode/FSNamesystem.java | 9 ++--- .../hdfs/server/namenode/NameNodeRpcServer.java | 4 +-- .../org/apache/hadoop/hdfs/tools/ECAdmin.java | 26 +------------- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 6 ++-- .../hdfs/ErasureCodeBenchmarkThroughput.java | 3 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 3 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 3 +- .../TestDFSStripedOutputStreamWithFailure.java | 2 +- .../hdfs/TestDecommissionWithStriped.java | 3 +- .../hadoop/hdfs/TestErasureCodingPolicies.java | 37 +++++++++++--------- .../TestErasureCodingPolicyWithSnapshot.java | 10 +++--- .../apache/hadoop/hdfs/TestFileChecksum.java | 3 +- .../hadoop/hdfs/TestFileStatusWithECPolicy.java | 2 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +- .../hdfs/TestReadStripedFileWithDecoding.java | 3 +- .../TestReadStripedFileWithMissingBlocks.java | 2 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 3 +- .../hdfs/TestSafeModeWithStripedFile.java | 3 +- .../TestUnsetAndChangeDirectoryEcPolicy.java | 18 +++++----- .../hadoop/hdfs/TestWriteReadStripedFile.java | 3 +- .../hdfs/TestWriteStripedFileWithFailure.java | 3 +- .../hdfs/server/balancer/TestBalancer.java | 3 +- .../TestBlockTokenWithDFSStriped.java | 4 +-- ...constructStripedBlocksWithRackAwareness.java | 9 +++-- .../TestSequentialBlockGroupId.java | 4 +-- .../TestDataNodeErasureCodingMetrics.java | 3 +- .../hadoop/hdfs/server/mover/TestMover.java | 3 +- .../TestAddOverReplicatedStripedBlocks.java | 3 +- .../namenode/TestAddStripedBlockInFBR.java | 3 +- .../server/namenode/TestAddStripedBlocks.java | 3 +- .../server/namenode/TestFSEditLogLoader.java | 4 +-- .../hdfs/server/namenode/TestFSImage.java | 6 ++-- .../hadoop/hdfs/server/namenode/TestFsck.java | 8 +++-- .../server/namenode/TestNameNodeMXBean.java | 3 +- .../namenode/TestQuotaWithStripedBlocks.java | 3 +- .../namenode/TestReconstructStripedBlocks.java | 6 ++-- .../server/namenode/TestStripedINodeFile.java | 6 ++-- .../TestOfflineImageViewer.java | 3 +- ...TestOfflineImageViewerWithStripedBlocks.java | 3 +- .../test/resources/testErasureCodingConf.xml | 2 +- .../java/org/apache/hadoop/fs/TestDFSIO.java | 2 +- 49 files changed, 152 insertions(+), 132 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index e0ccd62..ae1d821 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2611,12 +2611,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } - public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) + public void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException { checkOpen(); try (TraceScope ignored = newPathTraceScope("setErasureCodingPolicy", src)) { - namenode.setErasureCodingPolicy(src, ecPolicy); + namenode.setErasureCodingPolicy(src, ecPolicyName); } catch (RemoteException re) { throw re.unwrapRemoteException(AccessControlException.class, SafeModeException.class, http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 30dcfa4..1eef560 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2397,17 +2397,18 @@ public class DistributedFileSystem extends FileSystem { * Set the source path to the specified erasure coding policy. * * @param path The directory to set the policy - * @param ecPolicy The erasure coding policy. If not specified default will - * be used. + * @param ecPolicyName The erasure coding policy name. * @throws IOException */ public void setErasureCodingPolicy(final Path path, - final ErasureCodingPolicy ecPolicy) throws IOException { + final String ecPolicyName) throws IOException { Path absF = fixRelativePart(path); + Preconditions.checkNotNull(ecPolicyName, "Erasure coding policy cannot be" + + " null."); new FileSystemLinkResolver() { @Override public Void doCall(final Path p) throws IOException { - dfs.setErasureCodingPolicy(getPathName(p), ecPolicy); + dfs.setErasureCodingPolicy(getPathName(p), ecPolicyName); return null; } @@ -2415,7 +2416,7 @@ public class DistributedFileSystem extends FileSystem { public Void next(final FileSystem fs, final Path p) throws IOException { if (fs instanceof DistributedFileSystem) { DistributedFileSystem myDfs = (DistributedFileSystem) fs; - myDfs.setErasureCodingPolicy(p, ecPolicy); + myDfs.setErasureCodingPolicy(p, ecPolicyName); return null; } throw new UnsupportedOperationException( http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java index a192fa8..456f280 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java @@ -465,13 +465,15 @@ public class HdfsAdmin { * Set the source path to the specified erasure coding policy. * * @param path The source path referring to a directory. - * @param ecPolicy The erasure coding policy for the directory. - * If null, the default will be used. + * @param ecPolicyName The erasure coding policy name for the directory. + * * @throws IOException + * @throws HadoopIllegalArgumentException if the specified EC policy is not + * enabled on the cluster */ public void setErasureCodingPolicy(final Path path, - final ErasureCodingPolicy ecPolicy) throws IOException { - dfs.setErasureCodingPolicy(path, ecPolicy); + final String ecPolicyName) throws IOException { + dfs.setErasureCodingPolicy(path, ecPolicyName); } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index eaebd6f..041d226 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -1510,11 +1510,10 @@ public interface ClientProtocol { /** * Set an erasure coding policy on a specified path. * @param src The path to set policy on. - * @param ecPolicy The erasure coding policy. If null, default policy will - * be used + * @param ecPolicyName The erasure coding policy name. */ @AtMostOnce - void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) + void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException; /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java index de474b5..da4a17f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java @@ -1459,14 +1459,12 @@ public class ClientNamenodeProtocolTranslatorPB implements } @Override - public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) + public void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException { final SetErasureCodingPolicyRequestProto.Builder builder = SetErasureCodingPolicyRequestProto.newBuilder(); builder.setSrc(src); - if (ecPolicy != null) { - builder.setEcPolicy(PBHelperClient.convertErasureCodingPolicy(ecPolicy)); - } + builder.setEcPolicyName(ecPolicyName); SetErasureCodingPolicyRequestProto req = builder.build(); try { rpcProxy.setErasureCodingPolicy(null, req); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto index 7ea8d32..4eab4d3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto @@ -25,7 +25,7 @@ import "hdfs.proto"; message SetErasureCodingPolicyRequestProto { required string src = 1; - optional ErasureCodingPolicyProto ecPolicy = 2; + required string ecPolicyName = 2; } message SetErasureCodingPolicyResponseProto { http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java index 1aa15c8..1944fe7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java @@ -1442,9 +1442,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements RpcController controller, SetErasureCodingPolicyRequestProto req) throws ServiceException { try { - ErasureCodingPolicy ecPolicy = req.hasEcPolicy() ? - PBHelperClient.convertErasureCodingPolicy(req.getEcPolicy()) : null; - server.setErasureCodingPolicy(req.getSrc(), ecPolicy); + server.setErasureCodingPolicy(req.getSrc(), req.getEcPolicyName()); return SetErasureCodingPolicyResponseProto.newBuilder().build(); } catch (IOException e) { throw new ServiceException(e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java index 0ab8c89..f4c4ad7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java @@ -59,14 +59,16 @@ final class FSDirErasureCodingOp { * * @param fsn The namespace * @param srcArg The path of the target directory. - * @param ecPolicy The erasure coding policy to set on the target directory. + * @param ecPolicyName The erasure coding policy name to set on the target + * directory. * @param logRetryCache whether to record RPC ids in editlog for retry * cache rebuilding * @return {@link HdfsFileStatus} * @throws IOException + * @throws HadoopIllegalArgumentException if the policy is not enabled */ static HdfsFileStatus setErasureCodingPolicy(final FSNamesystem fsn, - final String srcArg, final ErasureCodingPolicy ecPolicy, + final String srcArg, final String ecPolicyName, final boolean logRetryCache) throws IOException { assert fsn.hasWriteLock(); @@ -78,6 +80,13 @@ final class FSDirErasureCodingOp { List xAttrs; fsd.writeLock(); try { + ErasureCodingPolicy ecPolicy = fsn.getErasureCodingPolicyManager() + .getPolicyByName(ecPolicyName); + if (ecPolicy == null) { + throw new HadoopIllegalArgumentException("Policy '" + + ecPolicyName + "' does not match any supported erasure coding " + + "policies."); + } iip = fsd.resolvePath(pc, src, DirOp.WRITE_LINK); src = iip.getPath(); xAttrs = setErasureCodingPolicyXAttr(fsn, iip, ecPolicy); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 38a326c..254ccfa 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -6772,13 +6772,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, /** * Set an erasure coding policy on the given path. * @param srcArg The path of the target directory. - * @param ecPolicy The erasure coding policy to set on the target directory. + * @param ecPolicyName The erasure coding policy to set on the target + * directory. * @throws AccessControlException if the caller is not the superuser. * @throws UnresolvedLinkException if the path can't be resolved. * @throws SafeModeException if the Namenode is in safe mode. */ - void setErasureCodingPolicy(final String srcArg, final ErasureCodingPolicy - ecPolicy, final boolean logRetryCache) throws IOException, + void setErasureCodingPolicy(final String srcArg, final String ecPolicyName, + final boolean logRetryCache) throws IOException, UnresolvedLinkException, SafeModeException, AccessControlException { final String operationName = "setErasureCodingPolicy"; checkSuperuserPrivilege(); @@ -6790,7 +6791,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, checkOperation(OperationCategory.WRITE); checkNameNodeSafeMode("Cannot set erasure coding policy on " + srcArg); resultingStat = FSDirErasureCodingOp.setErasureCodingPolicy(this, - srcArg, ecPolicy, logRetryCache); + srcArg, ecPolicyName, logRetryCache); success = true; } catch (AccessControlException ace) { logAuditEvent(success, operationName, srcArg, null, http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index f9cfa42..cf14e8a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -2018,7 +2018,7 @@ public class NameNodeRpcServer implements NamenodeProtocols { } @Override // ClientProtocol - public void setErasureCodingPolicy(String src, ErasureCodingPolicy ecPolicy) + public void setErasureCodingPolicy(String src, String ecPolicyName) throws IOException { checkNNStartup(); final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache); @@ -2027,7 +2027,7 @@ public class NameNodeRpcServer implements NamenodeProtocols { } boolean success = false; try { - namesystem.setErasureCodingPolicy(src, ecPolicy, cacheEntry != null); + namesystem.setErasureCodingPolicy(src, ecPolicyName, cacheEntry != null); success = true; } finally { RetryCache.setState(cacheEntry, success); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java index 29c65b1..548f754 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java @@ -28,7 +28,6 @@ import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; @@ -227,30 +226,7 @@ public class ECAdmin extends Configured implements Tool { final Path p = new Path(path); final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf); try { - ErasureCodingPolicy ecPolicy = null; - ErasureCodingPolicy[] ecPolicies = - dfs.getClient().getErasureCodingPolicies(); - for (ErasureCodingPolicy policy : ecPolicies) { - if (ecPolicyName.equals(policy.getName())) { - ecPolicy = policy; - break; - } - } - if (ecPolicy == null) { - StringBuilder sb = new StringBuilder(); - sb.append("Policy '"); - sb.append(ecPolicyName); - sb.append("' does not match any of the supported policies."); - sb.append(" Please select any one of "); - List ecPolicyNames = new ArrayList(); - for (ErasureCodingPolicy policy : ecPolicies) { - ecPolicyNames.add(policy.getName()); - } - sb.append(ecPolicyNames); - System.err.println(sb.toString()); - return 3; - } - dfs.setErasureCodingPolicy(p, ecPolicy); + dfs.setErasureCodingPolicy(p, ecPolicyName); System.out.println("Set erasure coding policy " + ecPolicyName + " on " + path); } catch (Exception e) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java index 1fbc1d9..60a1a4e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java @@ -136,6 +136,7 @@ import org.apache.hadoop.hdfs.server.datanode.DataNodeLayoutVersion; import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset; import org.apache.hadoop.hdfs.server.datanode.TestTransferRbw; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; +import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager; import org.apache.hadoop.hdfs.server.namenode.FSDirectory; import org.apache.hadoop.hdfs.server.namenode.FSEditLog; import org.apache.hadoop.hdfs.server.namenode.FSNamesystem; @@ -1898,7 +1899,7 @@ public class DFSTestUtil { Path dir, int numBlocks, int numStripesPerBlk, boolean toMkdir) throws Exception { createStripedFile(cluster, file, dir, numBlocks, numStripesPerBlk, - toMkdir, null); + toMkdir, ErasureCodingPolicyManager.getSystemDefaultPolicy()); } /** @@ -1922,7 +1923,8 @@ public class DFSTestUtil { assert dir != null; dfs.mkdirs(dir); try { - dfs.getClient().setErasureCodingPolicy(dir.toString(), ecPolicy); + dfs.getClient() + .setErasureCodingPolicy(dir.toString(), ecPolicy.getName()); } catch (IOException e) { if (!e.getMessage().contains("non-empty directory")) { throw e; http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java index 13dc997..d1a7569 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/ErasureCodeBenchmarkThroughput.java @@ -192,7 +192,8 @@ public class ErasureCodeBenchmarkThroughput } if (!dfs.exists(ecPath)) { dfs.mkdirs(ecPath); - dfs.getClient().setErasureCodingPolicy(ecPath.toString(), ecPolicy); + dfs.getClient() + .setErasureCodingPolicy(ecPath.toString(), ecPolicy.getName()); } else { Preconditions.checkArgument( dfs.getClient(). http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java index 9b69904..50b366b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java @@ -108,7 +108,8 @@ public class TestDFSStripedInputStream { } fs = cluster.getFileSystem(); fs.mkdirs(dirPath); - fs.getClient().setErasureCodingPolicy(dirPath.toString(), ecPolicy); + fs.getClient() + .setErasureCodingPolicy(dirPath.toString(), ecPolicy.getName()); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java index 8834e14..5956cc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java @@ -89,7 +89,8 @@ public class TestDFSStripedOutputStream { NativeRSRawErasureCoderFactory.class.getCanonicalName()); } cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", ecPolicy + .getName()); fs = cluster.getFileSystem(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java index fabd417..01bd3c5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java @@ -221,7 +221,7 @@ public class TestDFSStripedOutputStreamWithFailure { cluster.waitActive(); dfs = cluster.getFileSystem(); dfs.mkdirs(dir); - dfs.setErasureCodingPolicy(dir, ecPolicy); + dfs.setErasureCodingPolicy(dir, ecPolicy.getName()); } private void tearDown() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java index 2eab309..f8ed0c3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommissionWithStriped.java @@ -142,7 +142,8 @@ public class TestDecommissionWithStriped { client = getDfsClient(cluster.getNameNode(0), conf); dfs.mkdirs(ecDir); - dfs.setErasureCodingPolicy(ecDir, null); + dfs.setErasureCodingPolicy(ecDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java index 0e431fa..8608626 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java @@ -47,6 +47,8 @@ public class TestErasureCodingPolicies { private MiniDFSCluster cluster; private DistributedFileSystem fs; private static final int BLOCK_SIZE = 1024; + private static final ErasureCodingPolicy EC_POLICY = + ErasureCodingPolicyManager.getSystemDefaultPolicy(); private FSNamesystem namesystem; @Before @@ -80,7 +82,8 @@ public class TestErasureCodingPolicies { DFSTestUtil.createFile(fs, replicatedFile, 0, (short) 3, 0L); // set ec policy on dir - fs.setErasureCodingPolicy(dir, null); + fs.setErasureCodingPolicy(dir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); // create a file which should be using ec final Path ecSubDir = new Path(dir, "ecSubDir"); final Path ecFile = new Path(ecSubDir, "ecFile"); @@ -132,7 +135,7 @@ public class TestErasureCodingPolicies { fs.mkdir(testDir, FsPermission.getDirDefault()); /* Normal creation of an erasure coding directory */ - fs.getClient().setErasureCodingPolicy(testDir.toString(), null); + fs.setErasureCodingPolicy(testDir, EC_POLICY.getName()); /* Verify files under the directory are striped */ final Path ECFilePath = new Path(testDir, "foo"); @@ -148,7 +151,7 @@ public class TestErasureCodingPolicies { fs.mkdir(notEmpty, FsPermission.getDirDefault()); final Path oldFile = new Path(notEmpty, "old"); fs.create(oldFile); - fs.getClient().setErasureCodingPolicy(notEmpty.toString(), null); + fs.setErasureCodingPolicy(notEmpty, EC_POLICY.getName()); final Path newFile = new Path(notEmpty, "new"); fs.create(newFile); INode oldInode = namesystem.getFSDirectory().getINode(oldFile.toString()); @@ -160,10 +163,10 @@ public class TestErasureCodingPolicies { final Path dir1 = new Path("/dir1"); final Path dir2 = new Path(dir1, "dir2"); fs.mkdir(dir1, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(dir1.toString(), null); + fs.setErasureCodingPolicy(dir1, EC_POLICY.getName()); fs.mkdir(dir2, FsPermission.getDirDefault()); try { - fs.getClient().setErasureCodingPolicy(dir2.toString(), null); + fs.setErasureCodingPolicy(dir2, EC_POLICY.getName()); } catch (IOException e) { fail("Nested erasure coding policies are supported"); } @@ -172,7 +175,7 @@ public class TestErasureCodingPolicies { final Path fPath = new Path("/file"); fs.create(fPath); try { - fs.getClient().setErasureCodingPolicy(fPath.toString(), null); + fs.setErasureCodingPolicy(fPath, EC_POLICY.getName()); fail("Erasure coding policy on file"); } catch (IOException e) { assertExceptionContains("erasure coding policy for a file", e); @@ -185,8 +188,8 @@ public class TestErasureCodingPolicies { final Path dstECDir = new Path("/dstEC"); fs.mkdir(srcECDir, FsPermission.getDirDefault()); fs.mkdir(dstECDir, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(srcECDir.toString(), null); - fs.getClient().setErasureCodingPolicy(dstECDir.toString(), null); + fs.setErasureCodingPolicy(srcECDir, EC_POLICY.getName()); + fs.setErasureCodingPolicy(dstECDir, EC_POLICY.getName()); final Path srcFile = new Path(srcECDir, "foo"); fs.create(srcFile); @@ -220,7 +223,8 @@ public class TestErasureCodingPolicies { public void testReplication() throws IOException { final Path testDir = new Path("/ec"); fs.mkdir(testDir, FsPermission.getDirDefault()); - fs.setErasureCodingPolicy(testDir, null); + fs.setErasureCodingPolicy(testDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); final Path fooFile = new Path(testDir, "foo"); // create ec file with replication=0 fs.create(fooFile, FsPermission.getFileDefault(), true, @@ -241,8 +245,9 @@ public class TestErasureCodingPolicies { // dir EC policy should be null assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy()); // dir EC policy after setting - fs.getClient().setErasureCodingPolicy(src, null); //Default one will be used. - ErasureCodingPolicy sysDefaultECPolicy = ErasureCodingPolicyManager.getSystemDefaultPolicy(); + ErasureCodingPolicy sysDefaultECPolicy = + ErasureCodingPolicyManager.getSystemDefaultPolicy(); + fs.getClient().setErasureCodingPolicy(src, sysDefaultECPolicy.getName()); verifyErasureCodingInfo(src, sysDefaultECPolicy); fs.create(new Path(ecDir, "child1")).close(); // verify for the files in ec dir @@ -263,7 +268,7 @@ public class TestErasureCodingPolicies { // dir ECInfo before being set assertNull(fs.getClient().getFileInfo(src).getErasureCodingPolicy()); // dir ECInfo after set - fs.getClient().setErasureCodingPolicy(src, usingECPolicy); + fs.getClient().setErasureCodingPolicy(src, usingECPolicy.getName()); verifyErasureCodingInfo(src, usingECPolicy); fs.create(new Path(ecDir, "child1")).close(); // verify for the files in ec dir @@ -291,12 +296,12 @@ public class TestErasureCodingPolicies { final Path ecDir = new Path(src); try { fs.mkdir(ecDir, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(src, ecPolicy); + fs.getClient().setErasureCodingPolicy(src, ecPolicy.getName()); fail("HadoopIllegalArgumentException should be thrown for" + "setting an invalid erasure coding policy"); } catch (Exception e) { - assertExceptionContains("Policy [ RS-4-2-128k ] does not match " + - "any of the supported policies",e); + assertExceptionContains("Policy 'RS-4-2-128k' does not match " + + "any supported erasure coding policies",e); } } @@ -338,7 +343,7 @@ public class TestErasureCodingPolicies { for (ErasureCodingPolicy policy : sysPolicies) { Path dir = new Path("/policy_" + policy.getId()); fs.mkdir(dir, FsPermission.getDefault()); - fs.setErasureCodingPolicy(dir, policy); + fs.setErasureCodingPolicy(dir, policy.getName()); Path file = new Path(dir, "child"); fs.create(file).close(); assertEquals(policy, fs.getErasureCodingPolicy(file)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java index e02c0bf..6cf4ef4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicyWithSnapshot.java @@ -75,7 +75,7 @@ public class TestErasureCodingPolicyWithSnapshot { fs.mkdirs(ecDir); fs.allowSnapshot(ecDirParent); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); String contents = DFSTestUtil.readFile(fs, ecFile); final Path snap1 = fs.createSnapshot(ecDirParent, "snap1"); @@ -93,7 +93,7 @@ public class TestErasureCodingPolicyWithSnapshot { fs.getErasureCodingPolicy(snap2ECDir)); // Make dir again with system default ec policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); final Path snap3 = fs.createSnapshot(ecDirParent, "snap3"); final Path snap3ECDir = new Path(snap3, ecDir.getName()); // Check that snap3's ECPolicy has the correct settings @@ -134,7 +134,7 @@ public class TestErasureCodingPolicyWithSnapshot { fs.mkdirs(ecDir); fs.allowSnapshot(ecDir); - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, fs.getErasureCodingPolicy(snap1)); @@ -150,7 +150,7 @@ public class TestErasureCodingPolicyWithSnapshot { fs.allowSnapshot(ecDir); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); ErasureCodingPolicy ecSnap = fs.getErasureCodingPolicy(snap1); assertEquals("Got unexpected erasure coding policy", sysDefaultPolicy, @@ -182,7 +182,7 @@ public class TestErasureCodingPolicyWithSnapshot { fs.allowSnapshot(ecDir); // set erasure coding policy - fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy); + fs.setErasureCodingPolicy(ecDir, sysDefaultPolicy.getName()); DFSTestUtil.createFile(fs, ecFile, len, (short) 1, 0xFEED); final Path snap1 = fs.createSnapshot(ecDir, "snap1"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java index 9d60798..8ae176f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileChecksum.java @@ -81,7 +81,8 @@ public class TestFileChecksum { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); Path ecPath = new Path(ecDir); cluster.getFileSystem().mkdir(ecPath, FsPermission.getDirDefault()); - cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir, null); + cluster.getFileSystem().getClient().setErasureCodingPolicy(ecDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); client = fs.getClient(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java index 2210fbe..d7b7327 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileStatusWithECPolicy.java @@ -73,7 +73,7 @@ public class TestFileStatusWithECPolicy { final ErasureCodingPolicy ecPolicy1 = ErasureCodingPolicyManager.getSystemDefaultPolicy(); // set EC policy on dir - fs.setErasureCodingPolicy(dir, ecPolicy1); + fs.setErasureCodingPolicy(dir, ecPolicy1.getName()); final ErasureCodingPolicy ecPolicy2 = client.getFileInfo(dir.toUri().getPath()).getErasureCodingPolicy(); assertNotNull(ecPolicy2); assertTrue(ecPolicy1.equals(ecPolicy2)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java index 246818b..710ff62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestLeaseRecoveryStriped.java @@ -94,7 +94,7 @@ public class TestLeaseRecoveryStriped { cluster.waitActive(); dfs = cluster.getFileSystem(); dfs.mkdirs(dir); - dfs.setErasureCodingPolicy(dir, ecPolicy); + dfs.setErasureCodingPolicy(dir, ecPolicy.getName()); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java index 506a005..d7d9cc1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithDecoding.java @@ -102,7 +102,8 @@ public class TestReadStripedFileWithDecoding { conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_CONSIDERLOAD_KEY, false); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java index 2ee8663..b65626e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReadStripedFileWithMissingBlocks.java @@ -61,7 +61,7 @@ public class TestReadStripedFileWithMissingBlocks { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.getFileSystem().getClient().setErasureCodingPolicy( - "/", ecPolicy); + "/", ecPolicy.getName()); fs = cluster.getFileSystem(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java index 08b53c2..38939f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java @@ -107,7 +107,8 @@ public class TestReconstructStripedFile { cluster.waitActive(); fs = cluster.getFileSystem(); - fs.getClient().setErasureCodingPolicy("/", null); + fs.getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); List datanodes = cluster.getDataNodes(); for (int i = 0; i < dnNum; i++) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java index 2d37c06..0731779 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeModeWithStripedFile.java @@ -63,7 +63,8 @@ public class TestSafeModeWithStripedFile { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); conf.setLong(DFSConfigKeys.DFS_BLOCKREPORT_INTERVAL_MSEC_KEY, 100); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); cluster.waitActive(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java index f26aa06..86624cc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java @@ -101,7 +101,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { // Test unset a directory which has no EC policy fs.unsetErasureCodingPolicy(dirPath); // Set EC policy on directory - fs.setErasureCodingPolicy(dirPath, ecPolicy); + fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L); fs.unsetErasureCodingPolicy(dirPath); @@ -142,12 +142,12 @@ public class TestUnsetAndChangeDirectoryEcPolicy { .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID); fs.mkdirs(parentDir); - fs.setErasureCodingPolicy(parentDir, ecPolicy); + fs.setErasureCodingPolicy(parentDir, ecPolicy.getName()); fs.mkdirs(childDir); // Create RS(6,3) EC policy file DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L); // Set RS(3,2) EC policy on child directory - fs.setErasureCodingPolicy(childDir, ec32Policy); + fs.setErasureCodingPolicy(childDir, ec32Policy.getName()); // Create RS(3,2) EC policy file DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L); @@ -202,7 +202,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { // Test unset root path which has no EC policy fs.unsetErasureCodingPolicy(rootPath); // Set EC policy on root path - fs.setErasureCodingPolicy(rootPath, ecPolicy); + fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L); fs.unsetErasureCodingPolicy(rootPath); DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 1, 0L); @@ -240,11 +240,11 @@ public class TestUnsetAndChangeDirectoryEcPolicy { .getPolicyByPolicyID(HdfsConstants.RS_3_2_POLICY_ID); fs.unsetErasureCodingPolicy(rootPath); - fs.setErasureCodingPolicy(rootPath, ecPolicy); + fs.setErasureCodingPolicy(rootPath, ecPolicy.getName()); // Create RS(6,3) EC policy file DFSTestUtil.createFile(fs, ec63FilePath, fileLen, (short) 1, 0L); // Change EC policy from RS(6,3) to RS(3,2) - fs.setErasureCodingPolicy(rootPath, ec32Policy); + fs.setErasureCodingPolicy(rootPath, ec32Policy.getName()); DFSTestUtil.createFile(fs, ec32FilePath, fileLen, (short) 1, 0L); // start to check @@ -281,7 +281,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { final Path replicateFilePath2 = new Path(ecDirPath, "rep_file2"); fs.mkdirs(ecDirPath); - fs.setErasureCodingPolicy(ecDirPath, ecPolicy); + fs.setErasureCodingPolicy(ecDirPath, ecPolicy.getName()); DFSTestUtil.createFile(fs, ecFilePath, fileLen, (short) 1, 0L); fs.unsetErasureCodingPolicy(ecDirPath); DFSTestUtil.createFile(fs, replicateFilePath, fileLen, (short) 3, 0L); @@ -328,7 +328,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { // Set EC policy on non-existent directory try { - fs.setErasureCodingPolicy(dirPath, ecPolicy); + fs.setErasureCodingPolicy(dirPath, ecPolicy.getName()); fail("FileNotFoundException should be thrown for a non-existent" + " file path"); } catch (FileNotFoundException e) { @@ -347,7 +347,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy { // Set EC policy on file try { - fs.setErasureCodingPolicy(ecFilePath, ecPolicy); + fs.setErasureCodingPolicy(ecFilePath, ecPolicy.getName()); fail("IOException should be thrown for setting EC policy on file"); } catch (IOException e) { assertExceptionContains("Attempt to set an erasure coding policy " + http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java index 712f326..76ca704 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteReadStripedFile.java @@ -80,7 +80,8 @@ public class TestWriteReadStripedFile { cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); fs = cluster.getFileSystem(); fs.mkdirs(new Path("/ec")); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/ec", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java index b89e169..23a9821 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestWriteStripedFileWithFailure.java @@ -59,7 +59,8 @@ public class TestWriteStripedFileWithFailure { public void setup() throws IOException { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java index 1906d74..70aa4e0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java @@ -1940,7 +1940,8 @@ public class TestBalancer { cluster.waitActive(); client = NameNodeProxies.createProxy(conf, cluster.getFileSystem(0).getUri(), ClientProtocol.class).getProxy(); - client.setErasureCodingPolicy("/", null); + client.setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); long totalCapacity = sum(capacities); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java index 834df6f..20c3acc 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java @@ -83,8 +83,8 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS { .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100)) .numDataNodes(numDNs) .build(); - cluster.getFileSystem().getClient() - .setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); try { cluster.waitActive(); doTestRead(conf, cluster, true); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java index ace8732..832876e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReconstructStripedBlocksWithRackAwareness.java @@ -150,7 +150,8 @@ public class TestReconstructStripedBlocksWithRackAwareness { .numDataNodes(hosts.length).build(); cluster.waitActive(); fs = cluster.getFileSystem(); - fs.setErasureCodingPolicy(new Path("/"), null); + fs.setErasureCodingPolicy(new Path("/"), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); FSNamesystem fsn = cluster.getNamesystem(); BlockManager bm = fsn.getBlockManager(); @@ -220,7 +221,8 @@ public class TestReconstructStripedBlocksWithRackAwareness { .numDataNodes(hosts.length).build(); cluster.waitActive(); fs = cluster.getFileSystem(); - fs.setErasureCodingPolicy(new Path("/"), null); + fs.setErasureCodingPolicy(new Path("/"), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); MiniDFSCluster.DataNodeProperties lastHost = stopDataNode( hosts[hosts.length - 1]); @@ -273,7 +275,8 @@ public class TestReconstructStripedBlocksWithRackAwareness { .numDataNodes(hostNames.length).build(); cluster.waitActive(); fs = cluster.getFileSystem(); - fs.setErasureCodingPolicy(new Path("/"), null); + fs.setErasureCodingPolicy(new Path("/"), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); final BlockManager bm = cluster.getNamesystem().getBlockManager(); final DatanodeManager dm = bm.getDatanodeManager(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java index 10ea01f..7920c59 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestSequentialBlockGroupId.java @@ -88,8 +88,8 @@ public class TestSequentialBlockGroupId { blockGrpIdGenerator = cluster.getNamesystem().getBlockManager() .getBlockIdManager().getBlockGroupIdGenerator(); fs.mkdirs(ecDir); - cluster.getFileSystem().getClient() - .setErasureCodingPolicy("/ecDir", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/ecDir", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java index 7e64214..7036c7a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeErasureCodingMetrics.java @@ -75,7 +75,8 @@ public class TestDataNodeErasureCodingMetrics { conf.setInt(DFSConfigKeys.DFS_NAMENODE_REDUNDANCY_INTERVAL_SECONDS_KEY, 1); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java index d565548..a403ff4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java @@ -537,7 +537,8 @@ public class TestMover { client.setStoragePolicy(barDir, HdfsConstants.HOT_STORAGE_POLICY_NAME); // set an EC policy on "/bar" directory - client.setErasureCodingPolicy(barDir, null); + client.setErasureCodingPolicy(barDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); // write file to barDir final String fooFile = "/bar/foo"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java index c8b25564..670efd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddOverReplicatedStripedBlocks.java @@ -81,7 +81,8 @@ public class TestAddOverReplicatedStripedBlocks { cluster.waitActive(); fs = cluster.getFileSystem(); fs.mkdirs(dirPath); - fs.getClient().setErasureCodingPolicy(dirPath.toString(), null); + fs.getClient().setErasureCodingPolicy(dirPath.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java index 6e02372..87fbcc6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlockInFBR.java @@ -87,7 +87,8 @@ public class TestAddStripedBlockInFBR { final Path repDir = new Path("/rep"); dfs.mkdirs(ecDir); dfs.mkdirs(repDir); - dfs.getClient().setErasureCodingPolicy(ecDir.toString(), null); + dfs.getClient().setErasureCodingPolicy(ecDir.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); // create several non-EC files and one EC file final Path[] repFiles = new Path[groupSize]; http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java index ffd3fa7..2df1aa4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java @@ -86,7 +86,8 @@ public class TestAddStripedBlocks { .numDataNodes(groupSize).build(); cluster.waitActive(); dfs = cluster.getFileSystem(); - dfs.getClient().setErasureCodingPolicy("/", null); + dfs.getClient().setErasureCodingPolicy("/", ErasureCodingPolicyManager + .getSystemDefaultPolicy().getName()); } @After http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java index a57a936..72d76b7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSEditLogLoader.java @@ -479,7 +479,7 @@ public class TestFSEditLogLoader { //set the storage policy of the directory fs.mkdir(new Path(testDir), new FsPermission("755")); fs.getClient().getNamenode().setErasureCodingPolicy( - testDir, testECPolicy); + testDir, testECPolicy.getName()); // Create a file with striped block Path p = new Path(testFilePath); @@ -552,7 +552,7 @@ public class TestFSEditLogLoader { //set the storage policy of the directory fs.mkdir(new Path(testDir), new FsPermission("755")); fs.getClient().getNamenode().setErasureCodingPolicy( - testDir, testECPolicy); + testDir, testECPolicy.getName()); //create a file with striped blocks Path p = new Path(testFilePath); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java index 179e297..ae15491 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java @@ -156,7 +156,7 @@ public class TestFSImage { private void testSaveAndLoadStripedINodeFile(FSNamesystem fsn, Configuration conf, boolean isUC) throws IOException{ // Construct an INode with StripedBlock for saving and loading - fsn.setErasureCodingPolicy("/", testECPolicy, false); + fsn.setErasureCodingPolicy("/", testECPolicy.getName(), false); long id = 123456789; byte[] name = "testSaveAndLoadInodeFile_testfile".getBytes(); PermissionStatus permissionStatus = new PermissionStatus("testuser_a", @@ -472,8 +472,8 @@ public class TestFSImage { // Create directories and files fs.mkdirs(parentDir); fs.mkdirs(childDir); - fs.setErasureCodingPolicy(parentDir, testECPolicy); - fs.setErasureCodingPolicy(childDir, ec32Policy); + fs.setErasureCodingPolicy(parentDir, testECPolicy.getName()); + fs.setErasureCodingPolicy(childDir, ec32Policy.getName()); Path file_10_4 = new Path(parentDir, "striped_file_10_4"); Path file_3_2 = new Path(childDir, "striped_file_3_2"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java index c02e880..76c5378 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFsck.java @@ -690,7 +690,7 @@ public class TestFsck { util.createFiles(fs, topDir); // set topDir to EC when it has replicated files cluster.getFileSystem().getClient().setErasureCodingPolicy( - topDir, ecPolicy); + topDir, ecPolicy.getName()); // create a new file under topDir DFSTestUtil.createFile(fs, new Path(topDir, "ecFile"), 1024, (short) 1, 0L); @@ -2307,7 +2307,8 @@ public class TestFsck { // create file Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null); + fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); Path file = new Path(ecDirPath, "corrupted"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); @@ -2372,7 +2373,8 @@ public class TestFsck { // create file Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null); + fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); Path file = new Path(ecDirPath, "missing"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java index ef1e6e4..037fd40 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeMXBean.java @@ -738,7 +738,8 @@ public class TestNameNodeMXBean { // create file Path ecDirPath = new Path("/striped"); fs.mkdir(ecDirPath, FsPermission.getDirDefault()); - fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), null); + fs.getClient().setErasureCodingPolicy(ecDirPath.toString(), + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); Path file = new Path(ecDirPath, "corrupted"); final int length = cellSize * dataBlocks; final byte[] bytes = StripedFileTestUtil.generateBytes(length); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java index 9ec34d2..326ddc8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaWithStripedBlocks.java @@ -71,7 +71,8 @@ public class TestQuotaWithStripedBlocks { dfs = cluster.getFileSystem(); dfs.mkdirs(ecDir); - dfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy); + dfs.getClient() + .setErasureCodingPolicy(ecDir.toString(), ecPolicy.getName()); dfs.setQuota(ecDir, Long.MAX_VALUE - 1, DISK_QUOTA); dfs.setQuotaByStorageType(ecDir, StorageType.DISK, DISK_QUOTA); dfs.setStoragePolicy(ecDir, HdfsConstants.HOT_STORAGE_POLICY_NAME); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java index cf1585c..5e4a9db 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReconstructStripedBlocks.java @@ -201,7 +201,8 @@ public class TestReconstructStripedBlocks { cluster.waitActive(); DistributedFileSystem fs = cluster.getFileSystem(); BlockManager bm = cluster.getNamesystem().getBlockManager(); - fs.getClient().setErasureCodingPolicy("/", null); + fs.getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); int fileLen = dataBlocks * blockSize; Path p = new Path("/test2RecoveryTasksForSameBlockGroup"); final byte[] data = new byte[fileLen]; @@ -266,7 +267,8 @@ public class TestReconstructStripedBlocks { try { fs.mkdirs(dirPath); - fs.setErasureCodingPolicy(dirPath, null); + fs.setErasureCodingPolicy(dirPath, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); DFSTestUtil.createFile(fs, filePath, cellSize * dataBlocks * 2, (short) 1, 0L); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java index 03bd124..a712498 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestStripedINodeFile.java @@ -307,7 +307,8 @@ public class TestStripedINodeFile { dfs.mkdirs(ecDir); // set erasure coding policy - dfs.setErasureCodingPolicy(ecDir, null); + dfs.setErasureCodingPolicy(ecDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); DFSTestUtil.createFile(dfs, ecFile, len, (short) 1, 0xFEED); DFSTestUtil.createFile(dfs, contiguousFile, len, (short) 1, 0xFEED); final FSDirectory fsd = fsn.getFSDirectory(); @@ -408,7 +409,8 @@ public class TestStripedINodeFile { client.mkdirs(fooDir, new FsPermission((short) 777), true); client.setStoragePolicy(fooDir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME); // set an EC policy on "/foo" directory - client.setErasureCodingPolicy(fooDir, null); + client.setErasureCodingPolicy(fooDir, + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); // write file to fooDir final String barFile = "/foo/bar"; http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java index 053b856..b587017 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java @@ -236,7 +236,8 @@ public class TestOfflineImageViewer { ErasureCodingPolicy ecPolicy = ErasureCodingPolicyManager.getPolicyByPolicyID( HdfsConstants.XOR_2_1_POLICY_ID); - hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), ecPolicy); + hdfs.getClient().setErasureCodingPolicy(ecDir.toString(), + ecPolicy.getName()); writtenFiles.put(ecDir.toString(), hdfs.getFileStatus(ecDir)); // Create an empty Erasure Coded file http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java index db21217..e7794d6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewerWithStripedBlocks.java @@ -63,7 +63,8 @@ public class TestOfflineImageViewerWithStripedBlocks { conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build(); cluster.waitActive(); - cluster.getFileSystem().getClient().setErasureCodingPolicy("/", null); + cluster.getFileSystem().getClient().setErasureCodingPolicy("/", + ErasureCodingPolicyManager.getSystemDefaultPolicy().getName()); fs = cluster.getFileSystem(); Path eczone = new Path("/eczone"); fs.mkdirs(eczone); http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml index 9b9003a..278963a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml @@ -359,7 +359,7 @@ SubstringComparator - Policy 'invalidpolicy' does not match any of the supported policies. Please select any one of [ + Policy 'invalidpolicy' does not match any supported erasure coding policies. http://git-wip-us.apache.org/repos/asf/hadoop/blob/82ef9acc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java index d218169..2eb4b29 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java @@ -989,7 +989,7 @@ public class TestDFSIO implements Tool { ((DistributedFileSystem) fs).getAllErasureCodingPolicies(); for (ErasureCodingPolicy ec : list) { if (erasureCodePolicyName.equals(ec.getName())) { - ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec); + ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec.getName()); LOG.info("enable erasureCodePolicy = " + erasureCodePolicyName + " on " + path.toString()); break; --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org For additional commands, e-mail: common-commits-help@hadoop.apache.org