hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From inigo...@apache.org
Subject [09/50] [abbrv] hadoop git commit: Revert "HADOOP-13200. Implement customizable and configurable erasure coders. Contributed by Tim Yao."
Date Tue, 02 May 2017 22:05:26 GMT
Revert "HADOOP-13200. Implement customizable and configurable erasure coders. Contributed by
Tim Yao."

This reverts commit 872088c6e7e78534a8ffd1ea6e57b86faea3d6e1.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ddaeb3e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ddaeb3e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ddaeb3e4

Branch: refs/heads/HDFS-10467
Commit: ddaeb3e4979338551573285f219a2d361f502950
Parents: 872088c
Author: Wei-Chiu Chuang <weichiu@apache.org>
Authored: Thu Apr 27 12:25:47 2017 -0700
Committer: Wei-Chiu Chuang <weichiu@apache.org>
Committed: Thu Apr 27 12:25:47 2017 -0700

----------------------------------------------------------------------
 .../apache/hadoop/io/erasurecode/CodecUtil.java | 82 ++++++++++++++------
 .../io/erasurecode/ErasureCodeConstants.java    |  1 -
 .../rawcoder/DummyRawErasureCoderFactory.java   | 12 ---
 .../NativeRSRawErasureCoderFactory.java         | 13 ----
 .../NativeXORRawErasureCoderFactory.java        | 13 ----
 .../RSLegacyRawErasureCoderFactory.java         | 13 ----
 .../rawcoder/RSRawErasureCoderFactory.java      | 13 ----
 .../rawcoder/RawErasureCoderFactory.java        | 12 ---
 .../rawcoder/XORRawErasureCoderFactory.java     | 13 ----
 .../src/main/resources/core-default.xml         |  6 +-
 .../erasurecode/TestCodecRawCoderMapping.java   |  8 +-
 .../coder/TestHHXORErasureCoder.java            |  2 +-
 .../erasurecode/coder/TestRSErasureCoder.java   |  2 +-
 .../src/site/markdown/HDFSErasureCoding.md      | 14 ++--
 .../hadoop/hdfs/TestDFSStripedInputStream.java  |  2 +-
 .../hadoop/hdfs/TestDFSStripedOutputStream.java |  2 +-
 .../TestDFSStripedOutputStreamWithFailure.java  |  2 +-
 .../hadoop/hdfs/TestReconstructStripedFile.java |  2 +-
 .../TestUnsetAndChangeDirectoryEcPolicy.java    |  2 +-
 19 files changed, 76 insertions(+), 138 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
index 6cdbb37..c8b6a68 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecUtil.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.io.erasurecode;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableMap;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
@@ -28,12 +30,18 @@ import org.apache.hadoop.io.erasurecode.codec.RSErasureCodec;
 import org.apache.hadoop.io.erasurecode.codec.XORErasureCodec;
 import org.apache.hadoop.io.erasurecode.coder.ErasureDecoder;
 import org.apache.hadoop.io.erasurecode.coder.ErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory;
+import org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureCoderFactory;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
+import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
 
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
 
 /**
  * A codec & coder utility to help create coders conveniently.
@@ -71,12 +79,27 @@ public final class CodecUtil {
   /** Comma separated raw codec name. The first coder is prior to the latter. */
   public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_KEY =
       IO_ERASURECODE_CODEC + "rs-legacy.rawcoders";
+  public static final String IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT =
+      RSLegacyRawErasureCoderFactory.class.getCanonicalName();
   public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY =
       IO_ERASURECODE_CODEC + "rs.rawcoders";
+  public static final String IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT =
+      NativeRSRawErasureCoderFactory.class.getCanonicalName() +
+      "," + RSRawErasureCoderFactory.class.getCanonicalName();
 
   /** Raw coder factory for the XOR codec. */
   public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY =
       IO_ERASURECODE_CODEC + "xor.rawcoders";
+  public static final String IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT =
+      NativeXORRawErasureCoderFactory.class.getCanonicalName() +
+      "," + XORRawErasureCoderFactory.class.getCanonicalName();
+
+  // Default coders for each codec names.
+  public static final Map<String, String> DEFAULT_CODERS_MAP = ImmutableMap.of(
+      "rs",         IO_ERASURECODE_CODEC_RS_RAWCODERS_DEFAULT,
+      "rs-legacy",  IO_ERASURECODE_CODEC_RS_LEGACY_RAWCODERS_DEFAULT,
+      "xor",        IO_ERASURECODE_CODEC_XOR_RAWCODERS_DEFAULT
+  );
 
   private CodecUtil() { }
 
@@ -145,61 +168,70 @@ public final class CodecUtil {
   }
 
   private static RawErasureCoderFactory createRawCoderFactory(
-      String coderName, String codecName) {
+      Configuration conf, String rawCoderFactoryKey) {
     RawErasureCoderFactory fact;
-    fact = CodecRegistry.getInstance().
-            getCoderByName(codecName, coderName);
+    try {
+      Class<? extends RawErasureCoderFactory> factClass = conf.getClassByName(
+          rawCoderFactoryKey).asSubclass(RawErasureCoderFactory.class);
+      fact = factClass.newInstance();
+    } catch (ClassNotFoundException | InstantiationException |
+        IllegalAccessException e) {
+      throw new RuntimeException("Failed to create raw coder factory", e);
+    }
+
+    if (fact == null) {
+      throw new RuntimeException("Failed to create raw coder factory");
+    }
 
     return fact;
   }
 
-  // Return a list of coder names
-  private static String[] getRawCoderNames(
-      Configuration conf, String codecName) {
-    return conf.getStrings(
-      IO_ERASURECODE_CODEC + codecName + ".rawcoders",
-      CodecRegistry.getInstance().getCoderNames(codecName)
+  // Return comma separated coder names
+  private static String getRawCoders(Configuration conf, String codec) {
+    return conf.get(
+      IO_ERASURECODE_CODEC + codec + ".rawcoders",
+      DEFAULT_CODERS_MAP.getOrDefault(codec, codec)
     );
   }
 
   private static RawErasureEncoder createRawEncoderWithFallback(
-      Configuration conf, String codecName, ErasureCoderOptions coderOptions) {
-    String[] rawCoderNames = getRawCoderNames(conf, codecName);
-    for (String rawCoderName : rawCoderNames) {
+      Configuration conf, String codec, ErasureCoderOptions coderOptions) {
+    String coders = getRawCoders(conf, codec);
+    for (String factName : Splitter.on(",").split(coders)) {
       try {
-        if (rawCoderName != null) {
-          RawErasureCoderFactory fact = createRawCoderFactory(
-              rawCoderName, codecName);
+        if (factName != null) {
+          RawErasureCoderFactory fact = createRawCoderFactory(conf,
+              factName);
           return fact.createEncoder(coderOptions);
         }
       } catch (LinkageError | Exception e) {
         // Fallback to next coder if possible
-        LOG.warn("Failed to create raw erasure encoder " + rawCoderName +
+        LOG.warn("Failed to create raw erasure encoder " + factName +
             ", fallback to next codec if possible", e);
       }
     }
     throw new IllegalArgumentException("Fail to create raw erasure " +
-       "encoder with given codec: " + codecName);
+       "encoder with given codec: " + codec);
   }
 
   private static RawErasureDecoder createRawDecoderWithFallback(
-      Configuration conf, String codecName, ErasureCoderOptions coderOptions) {
-    String[] coders = getRawCoderNames(conf, codecName);
-    for (String rawCoderName : coders) {
+          Configuration conf, String codec, ErasureCoderOptions coderOptions) {
+    String coders = getRawCoders(conf, codec);
+    for (String factName : Splitter.on(",").split(coders)) {
       try {
-        if (rawCoderName != null) {
-          RawErasureCoderFactory fact = createRawCoderFactory(
-              rawCoderName, codecName);
+        if (factName != null) {
+          RawErasureCoderFactory fact = createRawCoderFactory(conf,
+              factName);
           return fact.createDecoder(coderOptions);
         }
       } catch (LinkageError | Exception e) {
         // Fallback to next coder if possible
-        LOG.warn("Failed to create raw erasure decoder " + rawCoderName +
+        LOG.warn("Failed to create raw erasure decoder " + factName +
             ", fallback to next codec if possible", e);
       }
     }
     throw new IllegalArgumentException("Fail to create raw erasure " +
-        "encoder with given codec: " + codecName);
+            "encoder with given codec: " + codec);
   }
 
   private static ErasureCodec createCodec(Configuration conf,

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index e0d7946..c830bb2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -25,7 +25,6 @@ public final class ErasureCodeConstants {
   private ErasureCodeConstants() {
   }
 
-  public static final String DUMMY_CODEC_NAME = "dummy";
   public static final String RS_CODEC_NAME = "rs";
   public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
   public static final String XOR_CODEC_NAME = "xor";

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
index 49b36b7..31ba4ef 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/DummyRawErasureCoderFactory.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.io.erasurecode.rawcoder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
@@ -26,7 +25,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
  */
 @InterfaceAudience.Private
 public class DummyRawErasureCoderFactory implements RawErasureCoderFactory {
-  public static final String CODER_NAME = "dummy_dummy";
 
   @Override
   public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
@@ -37,14 +35,4 @@ public class DummyRawErasureCoderFactory implements RawErasureCoderFactory
{
   public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
     return new DummyRawDecoder(coderOptions);
   }
-
-  @Override
-  public String getCoderName() {
-    return CODER_NAME;
-  }
-
-  @Override
-  public String getCodecName() {
-    return ErasureCodeConstants.DUMMY_CODEC_NAME;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
index cc98ec6..38997dd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeRSRawErasureCoderFactory.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.io.erasurecode.rawcoder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
@@ -28,8 +27,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 @InterfaceAudience.Private
 public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory {
 
-  public static final String CODER_NAME = "rs_native";
-
   @Override
   public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
     return new NativeRSRawEncoder(coderOptions);
@@ -39,14 +36,4 @@ public class NativeRSRawErasureCoderFactory implements RawErasureCoderFactory
{
   public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
     return new NativeRSRawDecoder(coderOptions);
   }
-
-  @Override
-  public String getCoderName() {
-    return CODER_NAME;
-  }
-
-  @Override
-  public String getCodecName() {
-    return ErasureCodeConstants.RS_CODEC_NAME;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
index 29a00dc..66b3f78 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/NativeXORRawErasureCoderFactory.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.io.erasurecode.rawcoder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
@@ -28,8 +27,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 @InterfaceAudience.Private
 public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory {
 
-  public static final String CODER_NAME = "xor_native";
-
   @Override
   public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
     return new NativeXORRawEncoder(coderOptions);
@@ -39,14 +36,4 @@ public class NativeXORRawErasureCoderFactory implements RawErasureCoderFactory
{
   public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
     return new NativeXORRawDecoder(coderOptions);
   }
-
-  @Override
-  public String getCoderName() {
-    return CODER_NAME;
-  }
-
-  @Override
-  public String getCodecName() {
-    return ErasureCodeConstants.XOR_CODEC_NAME;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java
index 6cbe33f..45bbf40 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSLegacyRawErasureCoderFactory.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.io.erasurecode.rawcoder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
@@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 @InterfaceAudience.Private
 public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory {
 
-  public static final String CODER_NAME = "rs-legacy_java";
-
   @Override
   public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
     return new RSLegacyRawEncoder(coderOptions);
@@ -38,14 +35,4 @@ public class RSLegacyRawErasureCoderFactory implements RawErasureCoderFactory
{
   public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
     return new RSLegacyRawDecoder(coderOptions);
   }
-
-  @Override
-  public String getCoderName() {
-    return CODER_NAME;
-  }
-
-  @Override
-  public String getCodecName() {
-    return ErasureCodeConstants.RS_LEGACY_CODEC_NAME;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
index 4aadcb7..8d954d5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RSRawErasureCoderFactory.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.io.erasurecode.rawcoder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
@@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 @InterfaceAudience.Private
 public class RSRawErasureCoderFactory implements RawErasureCoderFactory {
 
-  public static final String CODER_NAME = "rs_java";
-
   @Override
   public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
     return new RSRawEncoder(coderOptions);
@@ -38,14 +35,4 @@ public class RSRawErasureCoderFactory implements RawErasureCoderFactory
{
   public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
     return new RSRawDecoder(coderOptions);
   }
-
-  @Override
-  public String getCoderName() {
-    return CODER_NAME;
-  }
-
-  @Override
-  public String getCodecName() {
-    return ErasureCodeConstants.RS_CODEC_NAME;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
index 704e3e8..05f228b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoderFactory.java
@@ -41,16 +41,4 @@ public interface RawErasureCoderFactory {
    * @return raw erasure decoder
    */
   RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions);
-
-  /**
-   * Get the name of the coder.
-   * @return coder name
-   */
-  String getCoderName();
-
-  /**
-   * Get the name of its codec.
-   * @return codec name
-   */
-  String getCodecName();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
index 595f022..571fe12 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/XORRawErasureCoderFactory.java
@@ -18,7 +18,6 @@
 package org.apache.hadoop.io.erasurecode.rawcoder;
 
 import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
 import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 
 /**
@@ -27,8 +26,6 @@ import org.apache.hadoop.io.erasurecode.ErasureCoderOptions;
 @InterfaceAudience.Private
 public class XORRawErasureCoderFactory implements RawErasureCoderFactory {
 
-  public static final String CODER_NAME = "xor_java";
-
   @Override
   public RawErasureEncoder createEncoder(ErasureCoderOptions coderOptions) {
     return new XORRawEncoder(coderOptions);
@@ -38,14 +35,4 @@ public class XORRawErasureCoderFactory implements RawErasureCoderFactory
{
   public RawErasureDecoder createDecoder(ErasureCoderOptions coderOptions) {
     return new XORRawDecoder(coderOptions);
   }
-
-  @Override
-  public String getCoderName() {
-    return CODER_NAME;
-  }
-
-  @Override
-  public String getCodecName() {
-    return ErasureCodeConstants.XOR_CODEC_NAME;
-  }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index e9a4f1a..6fa70fd 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -668,7 +668,7 @@
 
 <property>
   <name>io.erasurecode.codec.rs.rawcoders</name>
-  <value>rs_native,rs_java</value>
+  <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeRSRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.RSRawErasureCoderFactory</value>
   <description>
     Comma separated raw coder implementations for the rs codec. The earlier
     factory is prior to followings in case of failure of creating raw coders.
@@ -677,7 +677,7 @@
 
 <property>
   <name>io.erasurecode.codec.rs-legacy.rawcoders</name>
-  <value>rs-legacy_java</value>
+  <value>org.apache.hadoop.io.erasurecode.rawcoder.RSLegacyRawErasureCoderFactory</value>
   <description>
     Comma separated raw coder implementations for the rs-legacy codec. The earlier
     factory is prior to followings in case of failure of creating raw coders.
@@ -686,7 +686,7 @@
 
 <property>
   <name>io.erasurecode.codec.xor.rawcoders</name>
-  <value>xor_native,xor_java</value>
+  <value>org.apache.hadoop.io.erasurecode.rawcoder.NativeXORRawErasureCoderFactory,org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory</value>
   <description>
     Comma separated raw coder implementations for the xor codec. The earlier
     factory is prior to followings in case of failure of creating raw coders.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
index 4a6fbd4..b5cada6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCodecRawCoderMapping.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureEncoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.XORRawDecoder;
 import org.apache.hadoop.io.erasurecode.rawcoder.XORRawEncoder;
-import org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory;
 import org.apache.hadoop.test.GenericTestUtils;
 import org.junit.Assert;
 import org.junit.Before;
@@ -105,8 +104,8 @@ public class TestCodecRawCoderMapping {
     ErasureCoderOptions coderOptions = new ErasureCoderOptions(
             numDataUnit, numParityUnit);
     conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
-        RSRawErasureCoderFactory.CODER_NAME +
-        "," + NativeRSRawErasureCoderFactory.CODER_NAME);
+        RSRawErasureCoderFactory.class.getCanonicalName() +
+        "," + NativeRSRawErasureCoderFactory.class.getCanonicalName());
     // should return default raw coder of rs codec
     RawErasureEncoder encoder = CodecUtil.createRawEncoder(
             conf, ErasureCodeConstants.RS_CODEC_NAME, coderOptions);
@@ -134,7 +133,8 @@ public class TestCodecRawCoderMapping {
     ErasureCoderOptions coderOptions = new ErasureCoderOptions(
             numDataUnit, numParityUnit);
     conf.set(CodecUtil.IO_ERASURECODE_CODEC_XOR_RAWCODERS_KEY,
-        "invalid-codec," + XORRawErasureCoderFactory.CODER_NAME);
+        "invalid-codec," +
+        "org.apache.hadoop.io.erasurecode.rawcoder.XORRawErasureCoderFactory");
     // should return second coder specified by IO_ERASURECODE_CODEC_CODERS
     RawErasureEncoder encoder = CodecUtil.createRawEncoder(
             conf, ErasureCodeConstants.XOR_CODEC_NAME, coderOptions);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
index 094ed08..a475a39 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestHHXORErasureCoder.java
@@ -51,7 +51,7 @@ public class TestHHXORErasureCoder extends TestHHErasureCoderBase {
      */
     Configuration conf = new Configuration();
     conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
-        RSRawErasureCoderFactory.CODER_NAME);
+        RSRawErasureCoderFactory.class.getCanonicalName());
     prepare(conf, 10, 4, new int[]{0}, new int[0]);
 
     testCoding(true);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
index 19054ce..11abac4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -58,7 +58,7 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
      */
     Configuration conf = new Configuration();
     conf.set(CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
-        RSRawErasureCoderFactory.CODER_NAME);
+        RSRawErasureCoderFactory.class.getCanonicalName());
     prepare(conf, 10, 4, new int[]{0}, new int[0]);
 
     testCoding(true);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index dbfb111..543b5ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -117,15 +117,11 @@ Deployment
   be more appropriate. If the administrator only cares about node-level fault-tolerance,
`RS-10-4-64k` would still be appropriate as long as
   there are at least 14 DataNodes in the cluster.
 
-  The codec implementations for Reed-Solomon and XOR can be configured with the following
client and DataNode configuration keys:
-  `io.erasurecode.codec.rs.rawcoders` for the default RS codec,
-  `io.erasurecode.codec.rs-legacy.rawcoders` for the legacy RS codec,
-  `io.erasurecode.codec.xor.rawcoders` for the XOR codec.
-  User can also configure self-defined codec with configuration key like:
-  `io.erasurecode.codec.self-defined-codec.rawcoders`.
-  The values for these key are lists of coder names with a fall-back mechanism.
-  All these codecs have implementations in pure Java. For default RS codec, there is also
a native implementation which leverages Intel ISA-L library to improve the performance of
codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve
the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for
more detail information.
-  The default implementation for RS Legacy is pure Java, and the default implementations
for default RS and XOR are native implementations using Intel ISA-L library.
+  The codec implementation for Reed-Solomon and XOR can be configured with the following
client and DataNode configuration keys:
+  `io.erasurecode.codec.rs.rawcoder` for the default RS codec,
+  `io.erasurecode.codec.rs-legacy.rawcoder` for the legacy RS codec,
+  `io.erasurecode.codec.xor.rawcoder` for the XOR codec.
+  The default implementations for all of these codecs are pure Java. For default RS codec,
there is also a native implementation which leverages Intel ISA-L library to improve the performance
of codec. For XOR codec, a native implementation which leverages Intel ISA-L library to improve
the performance of codec is also supported. Please refer to section "Enable Intel ISA-L" for
more detail information.
 
   Erasure coding background recovery work on the DataNodes can also be tuned via the following
configuration parameters:
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
index 4f67a0a..e3332fd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedInputStream.java
@@ -98,7 +98,7 @@ public class TestDFSStripedInputStream {
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
           CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
-          NativeRSRawErasureCoderFactory.CODER_NAME);
+          NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     SimulatedFSDataset.setFactory(conf);
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
index 14825ca..a2d5c8d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStream.java
@@ -85,7 +85,7 @@ public class TestDFSStripedOutputStream {
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
           CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
-          NativeRSRawErasureCoderFactory.CODER_NAME);
+          NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     DFSTestUtil.enableAllECPolicies(conf);
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
index 9915a2f..66a805c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSStripedOutputStreamWithFailure.java
@@ -214,7 +214,7 @@ public class TestDFSStripedOutputStreamWithFailure {
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
           CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
-          NativeRSRawErasureCoderFactory.CODER_NAME);
+          NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     DFSTestUtil.enableAllECPolicies(conf);
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
index 29e4028..2bd4a90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReconstructStripedFile.java
@@ -100,7 +100,7 @@ public class TestReconstructStripedFile {
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
           CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
-          NativeRSRawErasureCoderFactory.CODER_NAME);
+          NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     conf.set(DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_ENABLED_KEY,
         StripedFileTestUtil.getDefaultECPolicy().getName());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ddaeb3e4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
index 5371e20..6379db5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestUnsetAndChangeDirectoryEcPolicy.java
@@ -68,7 +68,7 @@ public class TestUnsetAndChangeDirectoryEcPolicy {
     if (ErasureCodeNative.isNativeCodeLoaded()) {
       conf.set(
           CodecUtil.IO_ERASURECODE_CODEC_RS_RAWCODERS_KEY,
-          NativeRSRawErasureCoderFactory.CODER_NAME);
+          NativeRSRawErasureCoderFactory.class.getCanonicalName());
     }
     DFSTestUtil.enableAllECPolicies(conf);
     cluster = new MiniDFSCluster.Builder(conf).numDataNodes(


---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org


Mime
View raw message