hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject [50/51] [abbrv] hadoop git commit: HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang.
Date Mon, 06 Apr 2015 17:38:50 GMT
HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bdb67d96
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bdb67d96
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bdb67d96

Branch: refs/heads/HDFS-7285
Commit: bdb67d96fbb0f23679f32e419bb17a7f8684411e
Parents: ea91d07
Author: Zhe Zhang <zhz@apache.org>
Authored: Fri Apr 3 15:22:50 2015 -0700
Committer: Zhe Zhang <zhz@apache.org>
Committed: Mon Apr 6 10:21:05 2015 -0700

----------------------------------------------------------------------
 .../coder/AbstractErasureDecoder.java           |  7 ++--
 .../coder/AbstractErasureEncoder.java           |  7 ++--
 .../io/erasurecode/coder/ErasureCoder.java      | 12 ++++++
 .../io/erasurecode/coder/ErasureDecoder.java    | 41 --------------------
 .../io/erasurecode/coder/ErasureEncoder.java    | 39 -------------------
 .../erasurecode/coder/TestErasureCoderBase.java | 20 +++++-----
 .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt        | 14 ++++++-
 7 files changed, 41 insertions(+), 99 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
index 54a6d1e..cd31294 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
@@ -23,13 +23,12 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
 /**
  * An abstract erasure decoder that's to be inherited by new decoders.
  *
- * It implements the {@link ErasureDecoder} interface.
+ * It implements the {@link ErasureCoder} interface.
  */
-public abstract class AbstractErasureDecoder extends AbstractErasureCoder
-    implements ErasureDecoder {
+public abstract class AbstractErasureDecoder extends AbstractErasureCoder {
 
   @Override
-  public ErasureCodingStep decode(ECBlockGroup blockGroup) {
+  public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
     // We may have more than this when considering complicate cases. HADOOP-11550
     return prepareDecodingStep(blockGroup);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
index 09b31e5..a836b75 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureEncoder.java
@@ -23,13 +23,12 @@ import org.apache.hadoop.io.erasurecode.ECBlockGroup;
 /**
  * An abstract erasure encoder that's to be inherited by new encoders.
  *
- * It implements the {@link ErasureEncoder} interface.
+ * It implements the {@link ErasureCoder} interface.
  */
-public abstract class AbstractErasureEncoder extends AbstractErasureCoder
-    implements ErasureEncoder {
+public abstract class AbstractErasureEncoder extends AbstractErasureCoder {
 
   @Override
-  public ErasureCodingStep encode(ECBlockGroup blockGroup) {
+  public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup) {
     // We may have more than this when considering complicate cases. HADOOP-11550
     return prepareEncodingStep(blockGroup);
   }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
index c5922f3..fb90156 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java
@@ -18,6 +18,7 @@
 package org.apache.hadoop.io.erasurecode.coder;
 
 import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.io.erasurecode.ECBlockGroup;
 
 /**
  * An erasure coder to perform encoding or decoding given a group. Generally it
@@ -64,6 +65,17 @@ public interface ErasureCoder extends Configurable {
   public int getChunkSize();
 
   /**
+   * Calculate the encoding or decoding steps given a block blockGroup.
+   *
+   * Note, currently only one coding step is supported. Will support complex
+   * cases of multiple coding steps.
+   *
+   * @param blockGroup the erasure coding block group containing all necessary
+   *                   information for codec calculation
+   */
+  public ErasureCodingStep calculateCoding(ECBlockGroup blockGroup);
+
+  /**
    * Tell if native or off-heap buffer is preferred or not. It's for callers to
    * decide how to allocate coding chunk buffers, either on heap or off heap.
    * It will return false by default.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
deleted file mode 100644
index dfd9e54..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureDecoder.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-
-import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-
-/**
- * Erasure decoder interface to perform decoding given a block group.
- *
- * It extends {@link ErasureCoder}.
- */
-public interface ErasureDecoder extends ErasureCoder {
-
-  /**
-   * Perform the decoding given a blockGroup. By default it will try the best to
-   * attempt to recover all the missing blocks according to the codec logic.
-   *
-   * Note, currently only one coding step is supported. Will support complex
-   * cases of multiple coding steps.
-   *
-   * @param blockGroup
-   */
-  public ErasureCodingStep decode(ECBlockGroup blockGroup);
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
deleted file mode 100644
index e837d22..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureEncoder.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.io.erasurecode.coder;
-
-import org.apache.hadoop.io.erasurecode.ECBlockGroup;
-
-/**
- * Erasure encoder interface to perform encoding given a block group.
- *
- * It extends {@link ErasureCoder}.
- */
-public interface ErasureEncoder extends ErasureCoder {
-
-  /**
-   * Calculate the encoding steps given a block blockGroup.
-   *
-   * Note, currently only one coding step is supported. Will support complex
-   * cases of multiple coding steps.
-   *
-   * @param blockGroup
-   */
-  public ErasureCodingStep encode(ECBlockGroup blockGroup);
-
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
index b963a59..05a62a7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
@@ -26,8 +26,8 @@ import org.apache.hadoop.io.erasurecode.TestCoderBase;
  * Erasure coder test base with utilities.
  */
 public abstract class TestErasureCoderBase extends TestCoderBase {
-  protected Class<? extends ErasureEncoder> encoderClass;
-  protected Class<? extends ErasureDecoder> decoderClass;
+  protected Class<? extends ErasureCoder> encoderClass;
+  protected Class<? extends ErasureCoder> decoderClass;
 
   protected int numChunksInBlock = 16;
 
@@ -55,7 +55,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
   protected void testCoding(boolean usingDirectBuffer) {
     this.usingDirectBuffer = usingDirectBuffer;
 
-    ErasureEncoder encoder = createEncoder();
+    ErasureCoder encoder = createEncoder();
 
     // Generate data and encode
     ECBlockGroup blockGroup = prepareBlockGroupForEncoding();
@@ -68,7 +68,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
 
     ErasureCodingStep codingStep;
     try {
-      codingStep = encoder.encode(blockGroup);
+      codingStep = encoder.calculateCoding(blockGroup);
       performCodingStep(codingStep);
     } finally {
       encoder.release();
@@ -78,9 +78,9 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
 
     //Decode
     blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks());
-    ErasureDecoder decoder = createDecoder();
+    ErasureCoder decoder = createDecoder();
     try {
-      codingStep = decoder.decode(blockGroup);
+      codingStep = decoder.calculateCoding(blockGroup);
       performCodingStep(codingStep);
     } finally {
       decoder.release();
@@ -138,8 +138,8 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
    * Create erasure encoder for test.
    * @return
    */
-  private ErasureEncoder createEncoder() {
-    ErasureEncoder encoder;
+  private ErasureCoder createEncoder() {
+    ErasureCoder encoder;
     try {
       encoder = encoderClass.newInstance();
     } catch (Exception e) {
@@ -155,8 +155,8 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
    * Create the erasure decoder for the test.
    * @return
    */
-  private ErasureDecoder createDecoder() {
-    ErasureDecoder decoder;
+  private ErasureCoder createDecoder() {
+    ErasureCoder decoder;
     try {
       decoder = decoderClass.newInstance();
     } catch (Exception e) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/bdb67d96/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index a686315..4e60a7c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -43,4 +43,16 @@
     blocks in NameNode (Jing Zhao)
 
     HDFS-8005. Erasure Coding: simplify striped block recovery work computation
-    and add tests (Jing Zhao)
\ No newline at end of file
+    and add tests (Jing Zhao)
+
+    HDFS-7617. Add unit tests for editlog transactions for EC 
+    (Hui Zheng via Zhe Zhang)
+
+    HADOOP-11782. Correct two thrown messages in ECSchema class
+    (Xinwei Qin via Kai Zheng)
+
+    HDFS-7839. Erasure coding: implement facilities in NameNode to create and
+    manage EC zones (Zhe Zhang)
+
+    HADOOP-11740. Combine erasure encoder and decoder interfaces (Zhe Zhang)
+


Mime
View raw message