hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dran...@apache.org
Subject hadoop git commit: HDFS-12448. Make sure user defined erasure coding policy ID will not overflow. Contributed by Huafeng Wang
Date Fri, 20 Oct 2017 01:38:03 GMT
Repository: hadoop
Updated Branches:
  refs/heads/trunk 0f1c03761 -> ce7cf66e5


HDFS-12448. Make sure user defined erasure coding policy ID will not overflow. Contributed
by Huafeng Wang


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce7cf66e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce7cf66e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce7cf66e

Branch: refs/heads/trunk
Commit: ce7cf66e5ed74c124afdb5a6902fbf297211cc04
Parents: 0f1c037
Author: Kai Zheng <kai.zheng@intel.com>
Authored: Fri Oct 20 09:42:04 2017 +0800
Committer: Kai Zheng <kai.zheng@intel.com>
Committed: Fri Oct 20 09:42:04 2017 +0800

----------------------------------------------------------------------
 .../io/erasurecode/ErasureCodeConstants.java    |  1 +
 .../namenode/ErasureCodingPolicyManager.java    | 20 +++++++++++----
 .../src/site/markdown/HDFSErasureCoding.md      |  2 +-
 .../hadoop/hdfs/TestErasureCodingPolicies.java  | 27 ++++++++++++++++++++
 4 files changed, 44 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce7cf66e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index d3c3b6b..73b8f56 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -50,6 +50,7 @@ public final class ErasureCodeConstants {
   public static final ECSchema REPLICATION_1_2_SCHEMA = new ECSchema(
       REPLICATION_CODEC_NAME, 1, 2);
 
+  public static final byte MAX_POLICY_ID = Byte.MAX_VALUE;
   public static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
   public static final byte REPLICATION_POLICY_ID = (byte) 63;
   public static final String REPLICATION_POLICY_NAME = REPLICATION_CODEC_NAME;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce7cf66e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 90699b4..62c7f60 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -253,6 +253,14 @@ public final class ErasureCodingPolicyManager {
         return p;
       }
     }
+
+    if (getCurrentMaxPolicyID() == ErasureCodeConstants.MAX_POLICY_ID) {
+      throw new HadoopIllegalArgumentException("Adding erasure coding " +
+          "policy failed because the number of policies stored in the " +
+          "system already reached the threshold, which is " +
+          ErasureCodeConstants.MAX_POLICY_ID);
+    }
+
     policy.setName(assignedNewName);
     policy.setId(getNextAvailablePolicyID());
     this.policiesByName.put(policy.getName(), policy);
@@ -261,12 +269,14 @@ public final class ErasureCodingPolicyManager {
     return policy;
   }
 
+  private byte getCurrentMaxPolicyID() {
+    return policiesByID.keySet().stream().max(Byte::compareTo).orElse((byte)0);
+  }
+
   private byte getNextAvailablePolicyID() {
-    byte currentId = this.policiesByID.keySet().stream()
-        .max(Byte::compareTo)
-        .filter(id -> id >= ErasureCodeConstants.USER_DEFINED_POLICY_START_ID)
-        .orElse(ErasureCodeConstants.USER_DEFINED_POLICY_START_ID);
-    return (byte) (currentId + 1);
+    byte nextPolicyID = (byte)(getCurrentMaxPolicyID() + 1);
+    return nextPolicyID > ErasureCodeConstants.USER_DEFINED_POLICY_START_ID ?
+        nextPolicyID : ErasureCodeConstants.USER_DEFINED_POLICY_START_ID;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce7cf66e/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index a171665..270201a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -193,7 +193,7 @@ Below are the details about each command.
 
  *  `[-addPolicies -policyFile <file>]`
 
-     Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template
for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize'
with the default value 4MB.
+     Add a list of erasure coding policies. Please refer etc/hadoop/user_ec_policies.xml.template
for the example policy file. The maximum cell size is defined in property 'dfs.namenode.ec.policies.max.cellsize'
with the default value 4MB. Currently HDFS allows the user to add 64 policies in total, and
the added policy ID is in range of 64 to 127. Adding policy will fail if there are already
64 policies added.
 
  *  `[-listCodecs]`
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce7cf66e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 2c2b05e..06ac073 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -752,6 +752,33 @@ public class TestErasureCodingPolicies {
   }
 
   @Test
+  public void testAddECPoliciesExceeded() throws Exception {
+    ECSchema toAddSchema = new ECSchema("rs", 3, 2);
+    int allowNumPolicies = ErasureCodeConstants.MAX_POLICY_ID -
+        ErasureCodeConstants.USER_DEFINED_POLICY_START_ID + 1;
+    for (int i = 0; i < allowNumPolicies; i++) {
+      ErasureCodingPolicy erasureCodingPolicy = new ErasureCodingPolicy(
+          toAddSchema, 1024 + 1024 * i);
+      ErasureCodingPolicy[] policyArray =
+          new ErasureCodingPolicy[]{erasureCodingPolicy};
+      AddErasureCodingPolicyResponse[] responses =
+          fs.addErasureCodingPolicies(policyArray);
+      assertEquals(1, responses.length);
+      assertTrue(responses[0].isSucceed());
+      assertEquals(responses[0].getPolicy().getId(),
+          ErasureCodeConstants.USER_DEFINED_POLICY_START_ID + i);
+    }
+    ErasureCodingPolicy erasureCodingPolicy = new ErasureCodingPolicy(
+        toAddSchema, 1024 + 1024 * allowNumPolicies);
+    ErasureCodingPolicy[] policyArray =
+        new ErasureCodingPolicy[]{erasureCodingPolicy};
+    AddErasureCodingPolicyResponse[] responses =
+        fs.addErasureCodingPolicies(policyArray);
+    assertEquals(1, responses.length);
+    assertFalse(responses[0].isSucceed());
+  }
+
+  @Test
   public void testReplicationPolicy() throws Exception {
     ErasureCodingPolicy replicaPolicy =
         SystemErasureCodingPolicies.getReplicationPolicy();


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message