hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From haiboc...@apache.org
Subject [06/50] [abbrv] hadoop git commit: HDFS-11793. Allow to enable user defined erasure coding policy. Contributed by Sammi Chen
Date Wed, 31 May 2017 22:42:41 GMT
HDFS-11793. Allow to enable user defined erasure coding policy. Contributed by Sammi Chen


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a62be38a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a62be38a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a62be38a

Branch: refs/heads/YARN-1011
Commit: a62be38a5e5d3a61dfb59054b3f5fd5b1b7053b3
Parents: 52661e0
Author: Kai Zheng <kai.zheng@intel.com>
Authored: Wed May 24 18:45:52 2017 +0800
Committer: Kai Zheng <kai.zheng@intel.com>
Committed: Wed May 24 18:45:52 2017 +0800

----------------------------------------------------------------------
 .../hadoop/io/erasurecode/CodecRegistry.java    | 19 ++----
 .../apache/hadoop/io/erasurecode/CodecUtil.java |  4 ++
 .../io/erasurecode/TestCodecRegistry.java       |  7 +-
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  4 +-
 .../hadoop/hdfs/DistributedFileSystem.java      |  9 ++-
 .../hdfs/protocol/AddECPolicyResponse.java      | 66 ++++++++++++++++++
 .../hdfs/protocol/AddingECPolicyResponse.java   | 66 ------------------
 .../hadoop/hdfs/protocol/ClientProtocol.java    |  7 +-
 .../hdfs/protocol/ErasureCodingPolicy.java      |  9 ++-
 .../ClientNamenodeProtocolTranslatorPB.java     | 10 +--
 .../hadoop/hdfs/protocolPB/PBHelperClient.java  | 22 +++---
 .../apache/hadoop/hdfs/util/ECPolicyLoader.java |  2 +-
 .../src/main/proto/erasurecoding.proto          |  2 +-
 .../src/main/proto/hdfs.proto                   |  2 +-
 .../hdfs/protocol/TestErasureCodingPolicy.java  | 10 +--
 .../src/main/conf/user_ec_policies.xml.template |  1 +
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |  2 +
 ...tNamenodeProtocolServerSideTranslatorPB.java |  8 +--
 .../namenode/ErasureCodingPolicyManager.java    | 72 ++++++++++++++------
 .../server/namenode/FSDirErasureCodingOp.java   |  4 +-
 .../hdfs/server/namenode/FSNamesystem.java      | 35 +++++++---
 .../hdfs/server/namenode/NameNodeRpcServer.java |  5 +-
 .../org/apache/hadoop/hdfs/tools/ECAdmin.java   |  6 +-
 .../src/main/resources/hdfs-default.xml         |  7 ++
 .../src/site/markdown/HDFSErasureCoding.md      |  3 +-
 .../hadoop/hdfs/TestDistributedFileSystem.java  | 39 -----------
 .../hadoop/hdfs/TestErasureCodingPolicies.java  | 54 +++++++++++++++
 .../hadoop/hdfs/protocolPB/TestPBHelper.java    | 26 +++----
 28 files changed, 289 insertions(+), 212 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java
index d53d598..fcf1349 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java
@@ -127,28 +127,20 @@ public final class CodecRegistry {
   /**
    * Get all coder names of the given codec.
    * @param codecName the name of codec
-   * @return an array of all coder names
+   * @return an array of all coder names, null if not exist
    */
   public String[] getCoderNames(String codecName) {
     String[] coderNames = coderNameMap.get(codecName);
-    if (coderNames == null) {
-      throw new IllegalArgumentException("No available raw coder factory for "
-          + codecName);
-    }
     return coderNames;
   }
 
   /**
    * Get all coder factories of the given codec.
    * @param codecName the name of codec
-   * @return a list of all coder factories
+   * @return a list of all coder factories, null if not exist
    */
   public List<RawErasureCoderFactory> getCoders(String codecName) {
     List<RawErasureCoderFactory> coders = coderMap.get(codecName);
-    if (coders == null) {
-      throw new IllegalArgumentException("No available raw coder factory for "
-          + codecName);
-    }
     return coders;
   }
 
@@ -164,7 +156,7 @@ public final class CodecRegistry {
    * Get a specific coder factory defined by codec name and coder name.
    * @param codecName name of the codec
    * @param coderName name of the coder
-   * @return the specific coder
+   * @return the specific coder, null if not exist
    */
   public RawErasureCoderFactory getCoderByName(
       String codecName, String coderName) {
@@ -176,10 +168,7 @@ public final class CodecRegistry {
         return coder;
       }
     }
-
-    // if not found, throw exception
-    throw new IllegalArgumentException("No implementation for coder "
-        + coderName + " of codec " + codecName);
+    return null;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 6cdbb37..75b8fa5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -153,6 +153,10 @@ public final class CodecUtil {
     return fact;
   }
 
+  public static boolean hasCodec(String codecName) {
+    return (CodecRegistry.getInstance().getCoderNames(codecName) != null);
+  }
+
   // Return a list of coder names
   private static String[] getRawCoderNames(
       Configuration conf, String codecName) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRegistry.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRegistry.java
index eb36557..5f17024 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRegistry.java
@@ -32,6 +32,7 @@ import java.util.List;
 import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -67,10 +68,11 @@ public class TestCodecRegistry {
     assertTrue(coders.get(1) instanceof XORRawErasureCoderFactory);
   }
 
-  @Test(expected = IllegalArgumentException.class)
+  @Test
   public void testGetCodersWrong() {
     List<RawErasureCoderFactory> coders =
         CodecRegistry.getInstance().getCoders("WRONG_CODEC");
+    assertNull(coders);
   }
 
   @Test
@@ -123,10 +125,11 @@ public class TestCodecRegistry {
     assertTrue(coder instanceof NativeXORRawErasureCoderFactory);
   }
 
-  @Test(expected = IllegalArgumentException.class)
+  @Test
   public void testGetCoderByNameWrong() {
     RawErasureCoderFactory coder = CodecRegistry.getInstance().
             getCoderByName(ErasureCodeConstants.RS_CODEC_NAME, "WRONG_RS");
+    assertNull(coder);
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 2e7dc1e..4fa7c5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -101,7 +101,7 @@ import org.apache.hadoop.hdfs.client.impl.DfsClientConf;
 import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
 import org.apache.hadoop.hdfs.net.Peer;
 import org.apache.hadoop.hdfs.protocol.AclException;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -2770,7 +2770,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
     }
   }
 
-  public AddingECPolicyResponse[] addErasureCodingPolicies(
+  public AddECPolicyResponse[] addErasureCodingPolicies(
       ErasureCodingPolicy[] policies) throws IOException {
     checkOpen();
     return namenode.addErasureCodingPolicies(policies);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 718c0b7..b65f9c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -71,7 +71,7 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
 import org.apache.hadoop.hdfs.client.impl.CorruptFileBlockIterator;
 import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -2549,13 +2549,16 @@ public class DistributedFileSystem extends FileSystem {
   }
 
   /**
-   * Add Erasure coding policies to HDFS.
+   * Add Erasure coding policies to HDFS. For each policy input, schema and
+   * cellSize are musts, name and id are ignored. They will be automatically
+   * created and assigned by Namenode once the policy is successfully added, and
+   * will be returned in the response.
    *
    * @param policies The user defined ec policy list to add.
    * @return Return the response list of adding operations.
    * @throws IOException
    */
-  public AddingECPolicyResponse[] addErasureCodingPolicies(
+  public AddECPolicyResponse[] addErasureCodingPolicies(
       ErasureCodingPolicy[] policies)  throws IOException {
     return dfs.addErasureCodingPolicies(policies);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java
new file mode 100644
index 0000000..e7a8435
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddECPolicyResponse.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+/**
+ * A response of add an ErasureCoding policy.
+ */
+public class AddECPolicyResponse {
+  private boolean succeed;
+  private ErasureCodingPolicy policy;
+  private String errorMsg;
+
+  public AddECPolicyResponse(ErasureCodingPolicy policy) {
+    this.policy = policy;
+    this.succeed = true;
+  }
+
+  public AddECPolicyResponse(ErasureCodingPolicy policy,
+      String errorMsg) {
+    this.policy = policy;
+    this.errorMsg = errorMsg;
+    this.succeed = false;
+  }
+
+  public AddECPolicyResponse(ErasureCodingPolicy policy,
+      IllegalECPolicyException e) {
+    this(policy, e.getMessage());
+  }
+
+  public boolean isSucceed() {
+    return succeed;
+  }
+
+  public ErasureCodingPolicy getPolicy() {
+    return policy;
+  }
+
+  public String getErrorMsg() {
+    return errorMsg;
+  }
+
+  @Override
+  public String toString() {
+    if (isSucceed()) {
+      return "Add ErasureCodingPolicy " + getPolicy().getName() + " succeed.";
+    } else {
+      return "Add ErasureCodingPolicy " + getPolicy().getName() + " failed and "
+          + "error message is " + getErrorMsg();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddingECPolicyResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddingECPolicyResponse.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddingECPolicyResponse.java
deleted file mode 100644
index ab39f09..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/AddingECPolicyResponse.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocol;
-
-/**
- * A response of adding an ErasureCoding policy.
- */
-public class AddingECPolicyResponse {
-  private boolean succeed;
-  private ErasureCodingPolicy policy;
-  private String errorMsg;
-
-  public AddingECPolicyResponse(ErasureCodingPolicy policy) {
-    this.policy = policy;
-    this.succeed = true;
-  }
-
-  public AddingECPolicyResponse(ErasureCodingPolicy policy,
-      String errorMsg) {
-    this.policy = policy;
-    this.errorMsg = errorMsg;
-    this.succeed = false;
-  }
-
-  public AddingECPolicyResponse(ErasureCodingPolicy policy,
-      IllegalECPolicyException e) {
-    this(policy, e.getMessage());
-  }
-
-  public boolean isSucceed() {
-    return succeed;
-  }
-
-  public ErasureCodingPolicy getPolicy() {
-    return policy;
-  }
-
-  public String getErrorMsg() {
-    return errorMsg;
-  }
-
-  @Override
-  public String toString() {
-    if (isSucceed()) {
-      return "Add ErasureCodingPolicy " + getPolicy().getName() + " succeed.";
-    } else {
-      return "Add ErasureCodingPolicy " + getPolicy().getName() + " failed and "
-          + "error message is " + getErrorMsg();
-    }
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 4b2b7d2..bf8eb4e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1526,14 +1526,17 @@ public interface ClientProtocol {
       throws IOException;
 
   /**
-   * Add Erasure coding policies.
+   * Add Erasure coding policies to HDFS. For each policy input, schema and
+   * cellSize are musts, name and id are ignored. They will be automatically
+   * created and assigned by Namenode once the policy is successfully added, and
+   * will be returned in the response.
    *
    * @param policies The user defined ec policy list to add.
    * @return Return the response list of adding operations.
    * @throws IOException
    */
   @AtMostOnce
-  AddingECPolicyResponse[] addErasureCodingPolicies(
+  AddECPolicyResponse[] addErasureCodingPolicies(
       ErasureCodingPolicy[] policies) throws IOException;
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
index 99bc4e6..b63d2c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
@@ -41,6 +41,8 @@ public final class ErasureCodingPolicy {
     Preconditions.checkNotNull(name);
     Preconditions.checkNotNull(schema);
     Preconditions.checkArgument(cellSize > 0, "cellSize must be positive");
+    Preconditions.checkArgument(cellSize % 1024 == 0,
+        "cellSize must be 1024 aligned");
     this.name = name;
     this.schema = schema;
     this.cellSize = cellSize;
@@ -51,8 +53,13 @@ public final class ErasureCodingPolicy {
     this(composePolicyName(schema, cellSize), schema, cellSize, id);
   }
 
+  public ErasureCodingPolicy(ECSchema schema, int cellSize) {
+    this(composePolicyName(schema, cellSize), schema, cellSize, (byte) -1);
+  }
+
   public static String composePolicyName(ECSchema schema, int cellSize) {
-    assert cellSize % 1024 == 0;
+    Preconditions.checkArgument(cellSize % 1024 == 0,
+        "cellSize must be 1024 aligned");
     return schema.getCodecName().toUpperCase() + "-" +
         schema.getNumDataUnits() + "-" + schema.getNumParityUnits() +
         "-" + cellSize / 1024 + "k";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 85bbc5f..19127d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import org.apache.hadoop.hdfs.AddBlockFlag;
 import org.apache.hadoop.hdfs.inotify.EventBatchList;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -1637,7 +1637,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
   }
 
   @Override
-  public AddingECPolicyResponse[] addErasureCodingPolicies(
+  public AddECPolicyResponse[] addErasureCodingPolicies(
       ErasureCodingPolicy[] policies) throws IOException {
     List<ErasureCodingPolicyProto> protos = Arrays.stream(policies)
         .map(PBHelperClient::convertErasureCodingPolicy)
@@ -1648,9 +1648,9 @@ public class ClientNamenodeProtocolTranslatorPB implements
     try {
       AddErasureCodingPoliciesResponseProto rep = rpcProxy
           .addErasureCodingPolicies(null, req);
-      AddingECPolicyResponse[] responses = rep.getResponsesList().stream()
-          .map(PBHelperClient::convertAddingECPolicyResponse)
-          .toArray(AddingECPolicyResponse[]::new);
+      AddECPolicyResponse[] responses = rep.getResponsesList().stream()
+          .map(PBHelperClient::convertAddECPolicyResponse)
+          .toArray(AddECPolicyResponse[]::new);
       return responses;
     }  catch (ServiceException e) {
       throw ProtobufHelper.getRemoteException(e);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index dc9a1ab..141a05a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -58,7 +58,7 @@ import org.apache.hadoop.hdfs.DFSUtilClient;
 import org.apache.hadoop.hdfs.inotify.Event;
 import org.apache.hadoop.hdfs.inotify.EventBatch;
 import org.apache.hadoop.hdfs.inotify.EventBatchList;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.BlockType;
@@ -124,7 +124,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmS
 import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.EncryptionZoneProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AccessModeProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AddingECPolicyResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AddECPolicyResponseProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockStoragePolicyProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTypeProto;
@@ -2709,10 +2709,10 @@ public class PBHelperClient {
     return builder.build();
   }
 
-  public static AddingECPolicyResponseProto convertAddingECPolicyResponse(
-      AddingECPolicyResponse response) {
-    AddingECPolicyResponseProto.Builder builder =
-        AddingECPolicyResponseProto.newBuilder()
+  public static AddECPolicyResponseProto convertAddECPolicyResponse(
+      AddECPolicyResponse response) {
+    AddECPolicyResponseProto.Builder builder =
+        AddECPolicyResponseProto.newBuilder()
         .setPolicy(convertErasureCodingPolicy(response.getPolicy()))
         .setSucceed(response.isSucceed());
     if (!response.isSucceed()) {
@@ -2721,13 +2721,13 @@ public class PBHelperClient {
     return builder.build();
   }
 
-  public static AddingECPolicyResponse convertAddingECPolicyResponse(
-      AddingECPolicyResponseProto proto) {
+  public static AddECPolicyResponse convertAddECPolicyResponse(
+      AddECPolicyResponseProto proto) {
     ErasureCodingPolicy policy = convertErasureCodingPolicy(proto.getPolicy());
     if (proto.getSucceed()) {
-      return new AddingECPolicyResponse(policy);
+      return new AddECPolicyResponse(policy);
     } else {
-      return new AddingECPolicyResponse(policy, proto.getErrorMsg());
+      return new AddECPolicyResponse(policy, proto.getErrorMsg());
     }
   }
 
@@ -2765,4 +2765,4 @@ public class PBHelperClient {
     }
     return ret;
   }
-}
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
index 02ae255..fb15926 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/ECPolicyLoader.java
@@ -316,7 +316,7 @@ public class ECPolicyLoader {
     }
 
     if (schema != null && cellSize > 0) {
-      return new ErasureCodingPolicy(schema, cellSize, (byte) -1);
+      return new ErasureCodingPolicy(schema, cellSize);
     } else {
       throw new RuntimeException("Bad policy is found in"
           + " EC policy configuration file");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
index 740d47d..ea878ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/erasurecoding.proto
@@ -58,7 +58,7 @@ message AddErasureCodingPoliciesRequestProto {
 }
 
 message AddErasureCodingPoliciesResponseProto {
-  repeated AddingECPolicyResponseProto responses = 1;
+  repeated AddECPolicyResponseProto responses = 1;
 }
 
 message UnsetErasureCodingPolicyRequestProto {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 08ed3c8..10ea5d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -371,7 +371,7 @@ message ErasureCodingPolicyProto {
   required uint32 id = 4; // Actually a byte - only 8 bits used
 }
 
-message AddingECPolicyResponseProto {
+message AddECPolicyResponseProto {
   required ErasureCodingPolicyProto policy = 1;
   required bool succeed = 2;
   optional string errorMsg = 3;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicy.java
index 643bbe7..17fb01c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/hdfs/protocol/TestErasureCodingPolicy.java
@@ -56,11 +56,11 @@ public class TestErasureCodingPolicy {
   @Test
   public void testEqualsAndHashCode() {
     ErasureCodingPolicy[] policies = new ErasureCodingPolicy[]{
-        new ErasureCodingPolicy("one", SCHEMA_1, 321, (byte) 1),
-        new ErasureCodingPolicy("two", SCHEMA_1, 321, (byte) 1),
-        new ErasureCodingPolicy("one", SCHEMA_2, 321, (byte) 1),
-        new ErasureCodingPolicy("one", SCHEMA_1, 123, (byte) 1),
-        new ErasureCodingPolicy("one", SCHEMA_1, 321, (byte) 3),
+        new ErasureCodingPolicy("one", SCHEMA_1, 1024, (byte) 1),
+        new ErasureCodingPolicy("two", SCHEMA_1, 1024, (byte) 1),
+        new ErasureCodingPolicy("one", SCHEMA_2, 1024, (byte) 1),
+        new ErasureCodingPolicy("one", SCHEMA_1, 2048, (byte) 1),
+        new ErasureCodingPolicy("one", SCHEMA_1, 1024, (byte) 3),
     };
 
     for (int i = 0; i < policies.length; i++) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/user_ec_policies.xml.template
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/user_ec_policies.xml.template b/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/user_ec_policies.xml.template
index 66546bb..8510b35 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/user_ec_policies.xml.template
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/user_ec_policies.xml.template
@@ -61,6 +61,7 @@
     <!-- schema is referred by its id -->
     <schema>XORk2m1</schema>
     <!-- cellsize must be an positive integer multiple of 1024(1k) -->
+    <!-- maximum cellsize is defined by 'dfs.namenode.ec.policies.max.cellsize' property -->
     <cellsize>131072</cellsize>
   </policy>
   <policy>

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b95c7e6..b3784b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -570,6 +570,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
 
   public static final String  DFS_NAMENODE_EC_POLICIES_ENABLED_KEY = "dfs.namenode.ec.policies.enabled";
   public static final String  DFS_NAMENODE_EC_POLICIES_ENABLED_DEFAULT = "";
+  public static final String  DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_KEY = "dfs.namenode.ec.policies.max.cellsize";
+  public static final int     DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT = 4 * 1024 * 1024;
   public static final String  DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_KEY = "dfs.datanode.ec.reconstruction.stripedread.threads";
   public static final int     DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_THREADS_DEFAULT = 20;
   public static final String  DFS_DN_EC_RECONSTRUCTION_STRIPED_READ_BUFFER_SIZE_KEY = "dfs.datanode.ec.reconstruction.stripedread.buffer.size";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index af4cd92..f10ce44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FsServerDefaults;
 import org.apache.hadoop.fs.Options.Rename;
 import org.apache.hadoop.fs.QuotaUsage;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
 import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -1649,11 +1649,11 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
       ErasureCodingPolicy[] policies = request.getEcPoliciesList().stream()
           .map(PBHelperClient::convertErasureCodingPolicy)
           .toArray(ErasureCodingPolicy[]::new);
-      AddingECPolicyResponse[] result = server
+      AddECPolicyResponse[] result = server
           .addErasureCodingPolicies(policies);
 
-      List<HdfsProtos.AddingECPolicyResponseProto> responseProtos = Arrays
-          .stream(result).map(PBHelperClient::convertAddingECPolicyResponse)
+      List<HdfsProtos.AddECPolicyResponseProto> responseProtos = Arrays
+          .stream(result).map(PBHelperClient::convertAddECPolicyResponse)
           .collect(Collectors.toList());
       AddErasureCodingPoliciesResponseProto response =
           AddErasureCodingPoliciesResponseProto.newBuilder()

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 4f27ed8..0feb79c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -25,7 +25,10 @@ import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 
-import java.util.List;
+import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
@@ -40,7 +43,12 @@ import java.util.stream.Stream;
  */
 @InterfaceAudience.LimitedPrivate({"HDFS"})
 public final class ErasureCodingPolicyManager {
-  private static final byte USER_DEFINED_POLICY_START_ID = 32;
+
+  public static Logger LOG = LoggerFactory.getLogger(
+      ErasureCodingPolicyManager.class);
+  private static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
+  private int maxCellSize =
+      DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT;
 
   // Supported storage policies for striped EC files
   private static final byte[] SUITABLE_STORAGE_POLICIES_FOR_EC_STRIPED_MODE =
@@ -77,10 +85,6 @@ public final class ErasureCodingPolicyManager {
   private ErasureCodingPolicyManager() {}
 
   public void init(Configuration conf) {
-    this.loadPolicies(conf);
-  }
-
-  private void loadPolicies(Configuration conf) {
     // Populate the list of enabled policies from configuration
     final String[] policyNames = conf.getTrimmedStrings(
         DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
@@ -95,20 +99,30 @@ public final class ErasureCodingPolicyManager {
       ErasureCodingPolicy ecPolicy =
           SystemErasureCodingPolicies.getByName(policyName);
       if (ecPolicy == null) {
-        String sysPolicies = SystemErasureCodingPolicies.getPolicies().stream()
-            .map(ErasureCodingPolicy::getName)
-            .collect(Collectors.joining(", "));
-        String msg = String.format("EC policy '%s' specified at %s is not a " +
-            "valid policy. Please choose from list of available policies: " +
-            "[%s]",
-            policyName,
-            DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
-            sysPolicies);
-        throw new IllegalArgumentException(msg);
+        ecPolicy = userPoliciesByName.get(policyName);
+        if (ecPolicy == null) {
+          String allPolicies = SystemErasureCodingPolicies.getPolicies()
+              .stream().map(ErasureCodingPolicy::getName)
+              .collect(Collectors.joining(", ")) + ", " +
+              userPoliciesByName.values().stream()
+              .map(ErasureCodingPolicy::getName)
+              .collect(Collectors.joining(", "));
+          String msg = String.format("EC policy '%s' specified at %s is not a "
+              + "valid policy. Please choose from list of available "
+              + "policies: [%s]",
+              policyName,
+              DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
+              allPolicies);
+          throw new IllegalArgumentException(msg);
+        }
       }
       enabledPoliciesByName.put(ecPolicy.getName(), ecPolicy);
     }
 
+    maxCellSize = conf.getInt(
+        DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_KEY,
+        DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT);
+
     /**
      * TODO: HDFS-7859 persist into NameNode
      * load persistent policies from image and editlog, which is done only once
@@ -153,9 +167,10 @@ public final class ErasureCodingPolicyManager {
    * Get all system defined policies and user defined policies.
    * @return all policies
    */
-  public List<ErasureCodingPolicy> getPolicies() {
+  public ErasureCodingPolicy[] getPolicies() {
     return Stream.concat(SystemErasureCodingPolicies.getPolicies().stream(),
-      this.userPoliciesByID.values().stream()).collect(Collectors.toList());
+        userPoliciesByName.values().stream())
+        .toArray(ErasureCodingPolicy[]::new);
   }
 
   /**
@@ -190,24 +205,37 @@ public final class ErasureCodingPolicyManager {
     // This is a placeholder for HDFS-7337.
   }
 
-  public synchronized void addPolicy(ErasureCodingPolicy policy)
+  public synchronized ErasureCodingPolicy addPolicy(ErasureCodingPolicy policy)
       throws IllegalECPolicyException {
+    if (!CodecUtil.hasCodec(policy.getCodecName())) {
+      throw new IllegalECPolicyException("Codec name "
+          + policy.getCodecName() + " is not supported");
+    }
+
+    if (policy.getCellSize() > maxCellSize) {
+      throw new IllegalECPolicyException("Cell size " + policy.getCellSize()
+          + " should not exceed maximum " + maxCellSize + " byte");
+    }
+
     String assignedNewName = ErasureCodingPolicy.composePolicyName(
         policy.getSchema(), policy.getCellSize());
     for (ErasureCodingPolicy p : getPolicies()) {
       if (p.getName().equals(assignedNewName)) {
-        throw new IllegalECPolicyException("The policy name already exists");
+        throw new IllegalECPolicyException("The policy name " + assignedNewName
+            + " already exists");
       }
       if (p.getSchema().equals(policy.getSchema()) &&
           p.getCellSize() == policy.getCellSize()) {
-        throw new IllegalECPolicyException("A policy with same schema and " +
-            "cell size already exists");
+        throw new IllegalECPolicyException("A policy with same schema "
+            + policy.getSchema().toString() + " and cell size "
+            + p.getCellSize() + " is already exists");
       }
     }
     policy.setName(assignedNewName);
     policy.setId(getNextAvailablePolicyID());
     this.userPoliciesByName.put(policy.getName(), policy);
     this.userPoliciesByID.put(policy.getId(), policy);
+    return policy;
   }
 
   private byte getNextAvailablePolicyID() {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 7b1cd2c..a875e4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -212,10 +212,10 @@ final class FSDirErasureCodingOp {
     return fsd.getAuditFileInfo(iip);
   }
 
-  static void addErasureCodePolicy(final FSNamesystem fsn,
+  static ErasureCodingPolicy addErasureCodePolicy(final FSNamesystem fsn,
       ErasureCodingPolicy policy) throws IllegalECPolicyException {
     Preconditions.checkNotNull(policy);
-    fsn.getErasureCodingPolicyManager().addPolicy(policy);
+    return fsn.getErasureCodingPolicyManager().addPolicy(policy);
   }
 
   private static List<XAttr> removeErasureCodingPolicyXAttr(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c17488d..15e52b7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -88,6 +88,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_PERMISSIONS_SUPERUSERGROU
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
 import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
 import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
+
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 import static org.apache.hadoop.util.Time.now;
 import static org.apache.hadoop.util.Time.monotonicNow;
@@ -175,7 +176,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
 import org.apache.hadoop.hdfs.HAUtil;
 import org.apache.hadoop.hdfs.HdfsConfiguration;
 import org.apache.hadoop.hdfs.UnknownCryptoProtocolVersionException;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.hdfs.protocol.Block;
 import org.apache.hadoop.hdfs.protocol.BlockType;
@@ -6847,21 +6848,33 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
    * @param policies The policies to add.
    * @return The according result of add operation.
    */
-  AddingECPolicyResponse[] addECPolicies(ErasureCodingPolicy[] policies)
+  AddECPolicyResponse[] addECPolicies(ErasureCodingPolicy[] policies)
       throws IOException {
+    final String operationName = "addECPolicies";
     checkOperation(OperationCategory.WRITE);
-    List<AddingECPolicyResponse> responses = new ArrayList<>();
+    List<AddECPolicyResponse> responses = new ArrayList<>();
+    boolean success = false;
     writeLock();
-    for (ErasureCodingPolicy policy : policies) {
-      try {
-        FSDirErasureCodingOp.addErasureCodePolicy(this, policy);
-        responses.add(new AddingECPolicyResponse(policy));
-      } catch (IllegalECPolicyException e) {
-        responses.add(new AddingECPolicyResponse(policy, e));
+    try {
+      checkOperation(OperationCategory.WRITE);
+      for (ErasureCodingPolicy policy : policies) {
+        try {
+          ErasureCodingPolicy newPolicy =
+              FSDirErasureCodingOp.addErasureCodePolicy(this, policy);
+          responses.add(new AddECPolicyResponse(newPolicy));
+        } catch (IllegalECPolicyException e) {
+          responses.add(new AddECPolicyResponse(policy, e));
+        }
       }
+      success = true;
+      return responses.toArray(new AddECPolicyResponse[0]);
+    } finally {
+      writeUnlock(operationName);
+      if (success) {
+        getEditLog().logSync();
+      }
+      logAuditEvent(success, operationName, null, null, null);
     }
-    writeUnlock("addECPolicies");
-    return responses.toArray(new AddingECPolicyResponse[0]);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index fba3427..4a4fe9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -84,7 +84,7 @@ import org.apache.hadoop.hdfs.HDFSPolicyProvider;
 import org.apache.hadoop.hdfs.inotify.EventBatch;
 import org.apache.hadoop.hdfs.inotify.EventBatchList;
 import org.apache.hadoop.hdfs.protocol.AclException;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
 import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
 import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
@@ -2270,9 +2270,10 @@ public class NameNodeRpcServer implements NamenodeProtocols {
   }
 
   @Override
-  public AddingECPolicyResponse[] addErasureCodingPolicies(
+  public AddECPolicyResponse[] addErasureCodingPolicies(
       ErasureCodingPolicy[] policies) throws IOException {
     checkNNStartup();
+    namesystem.checkSuperuserPrivilege();
     return namesystem.addECPolicies(policies);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 6508ec0..2041a05 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -22,7 +22,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.util.ECPolicyLoader;
 import org.apache.hadoop.tools.TableListing;
@@ -182,9 +182,9 @@ public class ECAdmin extends Configured implements Tool {
         List<ErasureCodingPolicy> policies =
             new ECPolicyLoader().loadPolicy(filePath);
         if (policies.size() > 0) {
-          AddingECPolicyResponse[] responses = dfs.addErasureCodingPolicies(
+          AddECPolicyResponse[] responses = dfs.addErasureCodingPolicies(
             policies.toArray(new ErasureCodingPolicy[policies.size()]));
-          for (AddingECPolicyResponse response : responses) {
+          for (AddECPolicyResponse response : responses) {
             System.out.println(response);
           }
         } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 9ddd343..82090e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2954,6 +2954,13 @@
 </property>
 
 <property>
+  <name>dfs.namenode.ec.policies.max.cellsize</name>
+  <value>4194304</value>
+  <description>The maximum cell size of erasure coding policy. Default is 4MB.
+  </description>
+</property>
+
+<property>
   <name>dfs.datanode.ec.reconstruction.stripedread.timeout.millis</name>
   <value>5000</value>
   <description>Datanode striped read timeout in milliseconds.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index fb2d56b..dcb56f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -154,6 +154,7 @@ Deployment
          [-getPolicy -path <path>]
          [-unsetPolicy -path <path>]
          [-listPolicies]
+         [-addPolicies -policyFile <file>]
          [-listCodecs]
          [-usage [cmd ...]]
          [-help [cmd ...]]
@@ -182,7 +183,7 @@ Below are the details about each command.
 
  *  `[-addPolicies -policyFile <file>]`
 
-     Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file.
+     Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize' with the default value 4MB.
 
  *  `[-listCodecs]`
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index f024fb6..e9af594 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -49,8 +49,6 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.BlockLocation;
@@ -77,18 +75,14 @@ import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
 import org.apache.hadoop.hdfs.client.impl.LeaseRenewer;
 import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
 import org.apache.hadoop.hdfs.net.Peer;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
-import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
 import org.apache.hadoop.hdfs.protocol.LocatedBlock;
-import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
 import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi;
 import org.apache.hadoop.hdfs.web.WebHdfsConstants;
-import org.apache.hadoop.io.erasurecode.ECSchema;
 import org.apache.hadoop.net.DNSToSwitchMapping;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.net.ScriptBasedMapping;
@@ -1449,37 +1443,4 @@ public class TestDistributedFileSystem {
       }
     }
   }
-
-  @Test
-  public void testAddErasureCodingPolicies() throws Exception {
-    Configuration conf = getTestConfiguration();
-    MiniDFSCluster cluster = null;
-
-    try {
-      ErasureCodingPolicy policy1 =
-          SystemErasureCodingPolicies.getPolicies().get(0);
-      conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
-          Stream.of(policy1).map(ErasureCodingPolicy::getName)
-          .collect(Collectors.joining(", ")));
-
-      cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
-      DistributedFileSystem fs = cluster.getFileSystem();
-
-      ECSchema toAddSchema = new ECSchema("testcodec", 3, 2);
-      ErasureCodingPolicy toAddPolicy =
-          new ErasureCodingPolicy(toAddSchema, 128 * 1024, (byte) 254);
-      ErasureCodingPolicy[] policies = new ErasureCodingPolicy[]{
-          policy1, toAddPolicy};
-      AddingECPolicyResponse[] responses =
-          fs.addErasureCodingPolicies(policies);
-      assertEquals(2, responses.length);
-      assertFalse(responses[0].isSucceed());
-      assertTrue(responses[1].isSucceed());
-      assertTrue(responses[1].getPolicy().getId() > 0);
-    } finally {
-      if (cluster != null) {
-        cluster.shutdown();
-      }
-    }
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 0e4bd29..77e6594 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -23,10 +23,12 @@ import org.apache.hadoop.fs.CreateFlag;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.DirectoryListing;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.protocol.HdfsConstants;
 import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
 import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
 import org.apache.hadoop.hdfs.server.namenode.INode;
 import org.apache.hadoop.hdfs.client.HdfsAdmin;
@@ -639,4 +641,56 @@ public class TestErasureCodingPolicies {
       }
     }
   }
+
+  @Test
+  public void testAddErasureCodingPolicies() throws Exception {
+    // Test nonexistent codec name
+    ECSchema toAddSchema = new ECSchema("testcodec", 3, 2);
+    ErasureCodingPolicy newPolicy =
+        new ErasureCodingPolicy(toAddSchema, 128 * 1024);
+    ErasureCodingPolicy[] policyArray = new ErasureCodingPolicy[]{newPolicy};
+    AddECPolicyResponse[] responses =
+        fs.addErasureCodingPolicies(policyArray);
+    assertEquals(1, responses.length);
+    assertFalse(responses[0].isSucceed());
+
+    // Test too big cell size
+    toAddSchema = new ECSchema("rs", 3, 2);
+    newPolicy =
+        new ErasureCodingPolicy(toAddSchema, 128 * 1024 * 1024);
+    policyArray = new ErasureCodingPolicy[]{newPolicy};
+    responses = fs.addErasureCodingPolicies(policyArray);
+    assertEquals(1, responses.length);
+    assertFalse(responses[0].isSucceed());
+
+    // Test other invalid cell size
+    toAddSchema = new ECSchema("rs", 3, 2);
+    int[] cellSizes = {0, -1, 1023};
+    for (int cellSize: cellSizes) {
+      try {
+        new ErasureCodingPolicy(toAddSchema, cellSize);
+        Assert.fail("Invalid cell size should be detected.");
+      } catch (Exception e){
+        GenericTestUtils.assertExceptionContains("cellSize must be", e);
+      }
+    }
+
+    // Test duplicate policy
+    ErasureCodingPolicy policy0 =
+        SystemErasureCodingPolicies.getPolicies().get(0);
+    policyArray  = new ErasureCodingPolicy[]{policy0};
+    responses = fs.addErasureCodingPolicies(policyArray);
+    assertEquals(1, responses.length);
+    assertFalse(responses[0].isSucceed());
+
+    // Test add policy successfully
+    newPolicy =
+        new ErasureCodingPolicy(toAddSchema, 1 * 1024 * 1024);
+    policyArray  = new ErasureCodingPolicy[]{newPolicy};
+    responses = fs.addErasureCodingPolicies(policyArray);
+    assertEquals(1, responses.length);
+    assertTrue(responses[0].isSucceed());
+    assertEquals(SystemErasureCodingPolicies.getPolicies().size() + 1,
+        ErasureCodingPolicyManager.getInstance().getPolicies().length);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a62be38a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 5d52609..3559825 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.hdfs.protocolPB;
 
 
 import com.google.protobuf.UninitializedMessageException;
-import org.apache.hadoop.hdfs.protocol.AddingECPolicyResponse;
+import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
 import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
 import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
 
@@ -913,14 +913,14 @@ public class TestPBHelper {
     // Check conversion of the built-in policies.
     for (ErasureCodingPolicy policy :
         SystemErasureCodingPolicies.getPolicies()) {
-      AddingECPolicyResponse response = new AddingECPolicyResponse(policy);
-      HdfsProtos.AddingECPolicyResponseProto proto = PBHelperClient
-          .convertAddingECPolicyResponse(response);
+      AddECPolicyResponse response = new AddECPolicyResponse(policy);
+      HdfsProtos.AddECPolicyResponseProto proto = PBHelperClient
+          .convertAddECPolicyResponse(response);
       // Optional fields should not be set.
       assertFalse("Unnecessary field is set.", proto.hasErrorMsg());
       // Convert proto back to an object and check for equality.
-      AddingECPolicyResponse convertedResponse = PBHelperClient
-          .convertAddingECPolicyResponse(proto);
+      AddECPolicyResponse convertedResponse = PBHelperClient
+          .convertAddECPolicyResponse(proto);
       assertEquals("Converted policy not equal", response.getPolicy(),
           convertedResponse.getPolicy());
       assertEquals("Converted policy not equal", response.isSucceed(),
@@ -929,13 +929,13 @@ public class TestPBHelper {
 
     ErasureCodingPolicy policy = SystemErasureCodingPolicies
         .getPolicies().get(0);
-    AddingECPolicyResponse response =
-        new AddingECPolicyResponse(policy, "failed");
-    HdfsProtos.AddingECPolicyResponseProto proto = PBHelperClient
-        .convertAddingECPolicyResponse(response);
+    AddECPolicyResponse response =
+        new AddECPolicyResponse(policy, "failed");
+    HdfsProtos.AddECPolicyResponseProto proto = PBHelperClient
+        .convertAddECPolicyResponse(response);
     // Convert proto back to an object and check for equality.
-    AddingECPolicyResponse convertedResponse = PBHelperClient
-        .convertAddingECPolicyResponse(proto);
+    AddECPolicyResponse convertedResponse = PBHelperClient
+        .convertAddECPolicyResponse(proto);
     assertEquals("Converted policy not equal", response.getPolicy(),
         convertedResponse.getPolicy());
     assertEquals("Converted policy not equal", response.getErrorMsg(),
@@ -961,7 +961,7 @@ public class TestPBHelper {
     // Check conversion of a non-built-in policy.
     ECSchema newSchema = new ECSchema("testcodec", 3, 2);
     ErasureCodingPolicy newPolicy =
-        new ErasureCodingPolicy(newSchema, 128 * 1024, (byte) 254);
+        new ErasureCodingPolicy(newSchema, 128 * 1024);
     HdfsProtos.ErasureCodingPolicyProto proto = PBHelperClient
         .convertErasureCodingPolicy(newPolicy);
     // Optional fields should be set.


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message