hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ji...@apache.org
Subject [45/50] hadoop git commit: HADOOP-11566. Add tests and fix for erasure coders to recover erased parity units. Contributed by Kai Zheng.
Date Tue, 19 May 2015 05:15:49 GMT
HADOOP-11566. Add tests and fix for erasure coders to recover erased parity units. Contributed
by Kai Zheng.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4dc91e58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4dc91e58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4dc91e58

Branch: refs/heads/HDFS-7285
Commit: 4dc91e58ed640da0dd00d4b4e2c3afcf18169151
Parents: 9be56c2
Author: Zhe Zhang <zhz@apache.org>
Authored: Mon May 18 10:13:03 2015 -0700
Committer: Jing Zhao <jing9@apache.org>
Committed: Mon May 18 22:11:11 2015 -0700

----------------------------------------------------------------------
 .../hadoop-common/CHANGES-HDFS-EC-7285.txt      |  3 ++
 .../apache/hadoop/io/erasurecode/ECChunk.java   | 17 ++++++-
 .../coder/AbstractErasureDecoder.java           | 13 ++++--
 .../hadoop/io/erasurecode/TestCoderBase.java    | 37 +++++++++++----
 .../erasurecode/coder/TestErasureCoderBase.java | 37 +++++++--------
 .../erasurecode/coder/TestRSErasureCoder.java   | 48 +++++++++++---------
 .../io/erasurecode/coder/TestXORCoder.java      |  6 +--
 .../io/erasurecode/rawcoder/TestRSRawCoder.java | 37 +++++++++------
 .../erasurecode/rawcoder/TestRawCoderBase.java  |  2 +-
 .../erasurecode/rawcoder/TestXORRawCoder.java   | 11 ++++-
 10 files changed, 134 insertions(+), 77 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index a152e31..34dfc9e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -48,3 +48,6 @@
     HADOOP-11921. Enhance tests for erasure coders. (Kai Zheng via Zhe Zhang)
 
     HADOOP-11920. Refactor some codes for erasure coders. (Kai Zheng via Zhe Zhang)
+
+    HADOOP-11566. Add tests and fix for erasure coders to recover erased parity 
+    units. (Kai Zheng via Zhe Zhang)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
index 436e13e..69a8343 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
@@ -58,8 +58,14 @@ public class ECChunk {
   public static ByteBuffer[] toBuffers(ECChunk[] chunks) {
     ByteBuffer[] buffers = new ByteBuffer[chunks.length];
 
+    ECChunk chunk;
     for (int i = 0; i < chunks.length; i++) {
-      buffers[i] = chunks[i].getBuffer();
+      chunk = chunks[i];
+      if (chunk == null) {
+        buffers[i] = null;
+      } else {
+        buffers[i] = chunk.getBuffer();
+      }
     }
 
     return buffers;
@@ -75,8 +81,15 @@ public class ECChunk {
     byte[][] bytesArr = new byte[chunks.length][];
 
     ByteBuffer buffer;
+    ECChunk chunk;
     for (int i = 0; i < chunks.length; i++) {
-      buffer = chunks[i].getBuffer();
+      chunk = chunks[i];
+      if (chunk == null) {
+        bytesArr[i] = null;
+        continue;
+      }
+
+      buffer = chunk.getBuffer();
       if (buffer.hasArray()) {
         bytesArr[i] = buffer.array();
       } else {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
index cd31294..6437236 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/AbstractErasureDecoder.java
@@ -60,16 +60,21 @@ public abstract class AbstractErasureDecoder extends AbstractErasureCoder
{
   }
 
   /**
-   * Which blocks were erased ? We only care data blocks here. Sub-classes can
-   * override this behavior.
+   * Which blocks were erased ?
    * @param blockGroup
    * @return output blocks to recover
    */
   protected ECBlock[] getOutputBlocks(ECBlockGroup blockGroup) {
-    ECBlock[] outputBlocks = new ECBlock[
-        getNumErasedBlocks(blockGroup.getDataBlocks())];
+    ECBlock[] outputBlocks = new ECBlock[getNumErasedBlocks(blockGroup)];
 
     int idx = 0;
+
+    for (int i = 0; i < getNumParityUnits(); i++) {
+      if (blockGroup.getParityBlocks()[i].isErased()) {
+        outputBlocks[idx++] = blockGroup.getParityBlocks()[i];
+      }
+    }
+
     for (int i = 0; i < getNumDataUnits(); i++) {
       if (blockGroup.getDataBlocks()[i].isErased()) {
         outputBlocks[idx++] = blockGroup.getDataBlocks()[i];

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
index 704b3f0..769427d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
@@ -37,10 +37,12 @@ public abstract class TestCoderBase {
   protected int numParityUnits;
   protected int chunkSize = 16 * 1024;
 
-  // Indexes of erased data units. Will also support test of erasing
-  // parity units
+  // Indexes of erased data units.
   protected int[] erasedDataIndexes = new int[] {0};
 
+  // Indexes of erased parity units.
+  protected int[] erasedParityIndexes = new int[] {0};
+
   // Data buffers are either direct or on-heap, for performance the two cases
   // may go to different coding implementations.
   protected boolean usingDirectBuffer = true;
@@ -52,12 +54,15 @@ public abstract class TestCoderBase {
    * @param erasedDataIndexes
    */
   protected void prepare(Configuration conf, int numDataUnits,
-                         int numParityUnits, int[] erasedDataIndexes) {
+                         int numParityUnits, int[] erasedDataIndexes,
+                         int[] erasedParityIndexes) {
     this.conf = conf;
     this.numDataUnits = numDataUnits;
     this.numParityUnits = numParityUnits;
     this.erasedDataIndexes = erasedDataIndexes != null ?
         erasedDataIndexes : new int[] {0};
+    this.erasedParityIndexes = erasedParityIndexes != null ?
+        erasedParityIndexes : new int[] {0};
   }
 
   /**
@@ -87,10 +92,15 @@ public abstract class TestCoderBase {
    * @return erased indexes altogether
    */
   protected int[] getErasedIndexesForDecoding() {
-    int[] erasedIndexesForDecoding = new int[erasedDataIndexes.length];
+    int[] erasedIndexesForDecoding =
+        new int[erasedParityIndexes.length + erasedDataIndexes.length];
 
     int idx = 0;
 
+    for (int i = 0; i < erasedParityIndexes.length; i++) {
+      erasedIndexesForDecoding[idx ++] = erasedParityIndexes[i];
+    }
+
     for (int i = 0; i < erasedDataIndexes.length; i++) {
       erasedIndexesForDecoding[idx ++] = erasedDataIndexes[i] + numParityUnits;
     }
@@ -123,15 +133,25 @@ public abstract class TestCoderBase {
    * Erase chunks to test the recovering of them. Before erasure clone them
    * first so could return them.
    * @param dataChunks
+   * @param parityChunks
    * @return clone of erased chunks
    */
-  protected ECChunk[] backupAndEraseChunks(ECChunk[] dataChunks) {
-    ECChunk[] toEraseChunks = new ECChunk[erasedDataIndexes.length];
+  protected ECChunk[] backupAndEraseChunks(ECChunk[] dataChunks,
+                                      ECChunk[] parityChunks) {
+    ECChunk[] toEraseChunks = new ECChunk[erasedParityIndexes.length +
+        erasedDataIndexes.length];
 
     int idx = 0;
+    ECChunk chunk;
+
+    for (int i = 0; i < erasedParityIndexes.length; i++) {
+      chunk = parityChunks[erasedParityIndexes[i]];
+      toEraseChunks[idx ++] = cloneChunkWithData(chunk);
+      eraseDataFromChunk(chunk);
+    }
 
     for (int i = 0; i < erasedDataIndexes.length; i++) {
-      ECChunk chunk = dataChunks[erasedDataIndexes[i]];
+      chunk = dataChunks[erasedDataIndexes[i]];
       toEraseChunks[idx ++] = cloneChunkWithData(chunk);
       eraseDataFromChunk(chunk);
     }
@@ -273,7 +293,8 @@ public abstract class TestCoderBase {
    * @return
    */
   protected ECChunk[] prepareOutputChunksForDecoding() {
-    ECChunk[] chunks = new ECChunk[erasedDataIndexes.length];
+    ECChunk[] chunks = new ECChunk[erasedDataIndexes.length +
+        erasedParityIndexes.length];
 
     for (int i = 0; i < chunks.length; i++) {
       chunks[i] = allocateOutputChunk();

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
index fdd0b50..f30323b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java
@@ -63,13 +63,15 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
     ECBlockGroup blockGroup = prepareBlockGroupForEncoding();
     // Backup all the source chunks for later recovering because some coders
     // may affect the source data.
-    TestBlock[] clonedDataBlocks = cloneBlocksWithData((TestBlock[]) blockGroup.getDataBlocks());
+    TestBlock[] clonedDataBlocks =
+        cloneBlocksWithData((TestBlock[]) blockGroup.getDataBlocks());
+    TestBlock[] parityBlocks = (TestBlock[]) blockGroup.getParityBlocks();
 
     ErasureCodingStep codingStep;
     codingStep = encoder.calculateCoding(blockGroup);
     performCodingStep(codingStep);
     // Erase specified sources but return copies of them for later comparing
-    TestBlock[] backupBlocks = backupAndEraseBlocks(clonedDataBlocks);
+    TestBlock[] backupBlocks = backupAndEraseBlocks(clonedDataBlocks, parityBlocks);
 
     // Decode
     blockGroup = new ECBlockGroup(clonedDataBlocks, blockGroup.getParityBlocks());
@@ -207,34 +209,27 @@ public abstract class TestErasureCoderBase extends TestCoderBase {
    * @param dataBlocks
    * @return clone of erased dataBlocks
    */
-  protected TestBlock[] backupAndEraseBlocks(TestBlock[] dataBlocks) {
-    TestBlock[] toEraseBlocks = new TestBlock[erasedDataIndexes.length];
+  protected TestBlock[] backupAndEraseBlocks(TestBlock[] dataBlocks,
+                                             TestBlock[] parityBlocks) {
+    TestBlock[] toEraseBlocks = new TestBlock[erasedDataIndexes.length +
+                                          erasedParityIndexes.length];
 
     int idx = 0;
+    TestBlock block;
 
-    for (int i = 0; i < erasedDataIndexes.length; i++) {
-      TestBlock block = dataBlocks[erasedDataIndexes[i]];
+    for (int i = 0; i < erasedParityIndexes.length; i++) {
+      block = parityBlocks[erasedParityIndexes[i]];
       toEraseBlocks[idx ++] = cloneBlockWithData(block);
       eraseDataFromBlock(block);
     }
 
-    return toEraseBlocks;
-  }
-
-  /**
-   * Copy those data blocks that's to be erased for later comparing and
-   * verifying.
-   * @param dataBlocks
-   * @return
-   */
-  protected TestBlock[] copyDataBlocksToErase(TestBlock[] dataBlocks) {
-    TestBlock[] copiedBlocks = new TestBlock[erasedDataIndexes.length];
-
-    for (int i = 0; i < erasedDataIndexes.length; ++i) {
-      copiedBlocks[i] = cloneBlockWithData(dataBlocks[erasedDataIndexes[i]]);
+    for (int i = 0; i < erasedDataIndexes.length; i++) {
+      block = dataBlocks[erasedDataIndexes[i]];
+      toEraseBlocks[idx ++] = cloneBlockWithData(block);
+      eraseDataFromBlock(block);
     }
 
-    return copiedBlocks;
+    return toEraseBlocks;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
index 3ae6a93..94f77db 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestRSErasureCoder.java
@@ -40,8 +40,8 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
   }
 
   @Test
-  public void testCodingNoDirectBuffer_10x4_erasing_d0() {
-    prepare(null, 10, 4, new int[] {0});
+  public void testCodingNoDirectBuffer_10x4_erasing_d0_p0() {
+    prepare(null, 10, 4, new int[] {0}, new int[] {0});
     /**
      * Doing twice to test if the coders can be repeatedly reused. This matters
      * as the underlying coding buffers are shared, which may have bugs.
@@ -53,34 +53,41 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
   @Test
   public void testCodingDirectBufferWithConf_10x4_erasing_d0() {
     /**
-     * This tests if the two configuration items work or not.
+     * This tests if the configuration items work or not.
      */
     Configuration conf = new Configuration();
-    conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY, RSRawErasureCoderFactory.class.getCanonicalName());
-
-    prepare(conf, 10, 4, new int[]{0});
+    conf.set(CommonConfigurationKeys.IO_ERASURECODE_CODEC_RS_RAWCODER_KEY,
+        RSRawErasureCoderFactory.class.getCanonicalName());
+    prepare(conf, 10, 4, new int[]{0}, new int[0]);
 
     testCoding(true);
     testCoding(true);
   }
-
+  
+  @Test
+  public void testCodingDirectBuffer_10x4_erasing_p1() {
+    prepare(null, 10, 4, new int[]{}, new int[]{1});
+    testCoding(true);
+    testCoding(true);
+  }
+  
   @Test
   public void testCodingDirectBuffer_10x4_erasing_d2() {
-    prepare(null, 10, 4, new int[] {2});
+    prepare(null, 10, 4, new int[] {2}, new int[] {});
     testCoding(true);
     testCoding(true);
   }
 
   @Test
-  public void testCodingDirectBuffer_10x4_erasing_d0() {
-    prepare(null, 10, 4, new int[] {0});
+  public void testCodingDirectBuffer_10x4_erasing_d0_p0() {
+    prepare(null, 10, 4, new int[] {0}, new int[] {0});
     testCoding(true);
     testCoding(true);
   }
 
   @Test
-  public void testCodingBothBuffers_10x4_erasing_d0() {
-    prepare(null, 10, 4, new int[] {0});
+  public void testCodingBothBuffers_10x4_erasing_d0_p0() {
+    prepare(null, 10, 4, new int[] {0}, new int[] {0});
 
     /**
      * Doing in mixed buffer usage model to test if the coders can be repeatedly
@@ -94,27 +101,26 @@ public class TestRSErasureCoder extends TestErasureCoderBase {
   }
 
   @Test
-  public void testCodingDirectBuffer_10x4_erasure_of_d2_d4() {
-    prepare(null, 10, 4, new int[] {2, 4});
+  public void testCodingDirectBuffer_10x4_erasure_of_d2_d4_p0() {
+    prepare(null, 10, 4, new int[] {2, 4}, new int[] {0});
     testCoding(true);
   }
 
   @Test
-  public void testCodingDirectBuffer_10x4_erasing_d0_d1() {
-    prepare(null, 10, 4, new int[] {0, 1});
+  public void testCodingDirectBuffer_10x4_erasing_d0_d1_p0_p1() {
+    prepare(null, 10, 4, new int[] {0, 1}, new int[] {0, 1});
     testCoding(true);
   }
 
   @Test
-  public void testCodingNoDirectBuffer_3x3_erasing_d0() {
-    prepare(null, 3, 3, new int[] {0});
+  public void testCodingNoDirectBuffer_3x3_erasing_d0_p0() {
+    prepare(null, 3, 3, new int[] {0}, new int[] {0});
     testCoding(false);
   }
 
   @Test
-  public void testCodingDirectBuffer_3x3_erasing_d0() {
-    prepare(null, 3, 3, new int[] {0});
+  public void testCodingDirectBuffer_3x3_erasing_d0_p0() {
+    prepare(null, 3, 3, new int[] {0}, new int[] {0});
     testCoding(true);
   }
-
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java
index 87aa656..06e0087 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestXORCoder.java
@@ -36,8 +36,8 @@ public class TestXORCoder extends TestErasureCoderBase {
   }
 
   @Test
-  public void testCodingNoDirectBuffer_erasing_d0() {
-    prepare(null, 10, 1, new int[] {0});
+  public void testCodingNoDirectBuffer_erasing_p0() {
+    prepare(null, 10, 1, new int[0], new int[] {0});
 
     /**
      * Doing twice to test if the coders can be repeatedly reused. This matters
@@ -49,7 +49,7 @@ public class TestXORCoder extends TestErasureCoderBase {
 
   @Test
   public void testCodingBothBuffers_erasing_d5() {
-    prepare(null, 10, 1, new int[]{5});
+    prepare(null, 10, 1, new int[]{5}, new int[0]);
 
     /**
      * Doing in mixed buffer usage model to test if the coders can be repeatedly

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
index 9ba3e88..84bad92 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
@@ -32,8 +32,8 @@ public class TestRSRawCoder extends TestRSRawCoderBase {
   }
 
   @Test
-  public void testCodingNoDirectBuffer_10x4_erasing_d0() {
-    prepare(null, 10, 4, new int[] {0});
+  public void testCodingNoDirectBuffer_10x4_erasing_d0_p0() {
+    prepare(null, 10, 4, new int[] {0}, new int[] {0});
     /**
      * Doing twice to test if the coders can be repeatedly reused. This matters
      * as the underlying coding buffers are shared, which may have bugs.
@@ -43,22 +43,29 @@ public class TestRSRawCoder extends TestRSRawCoderBase {
   }
 
   @Test
+  public void testCodingDirectBuffer_10x4_erasing_p1() {
+    prepare(null, 10, 4, new int[] {}, new int[] {1});
+    testCoding(true);
+    testCoding(true);
+  }
+
+  @Test
   public void testCodingDirectBuffer_10x4_erasing_d2() {
-    prepare(null, 10, 4, new int[] {2});
+    prepare(null, 10, 4, new int[] {2}, new int[] {});
     testCoding(true);
     testCoding(true);
   }
 
   @Test
-  public void testCodingDirectBuffer_10x4_erasing_d0() {
-    prepare(null, 10, 4, new int[] {0});
+  public void testCodingDirectBuffer_10x4_erasing_d0_p0() {
+    prepare(null, 10, 4, new int[] {0}, new int[] {0});
     testCoding(true);
     testCoding(true);
   }
 
   @Test
-  public void testCodingBothBuffers_10x4_erasing_d0() {
-    prepare(null, 10, 4, new int[] {0});
+  public void testCodingBothBuffers_10x4_erasing_d0_p0() {
+    prepare(null, 10, 4, new int[] {0}, new int[] {0});
 
     /**
      * Doing in mixed buffer usage model to test if the coders can be repeatedly
@@ -72,26 +79,26 @@ public class TestRSRawCoder extends TestRSRawCoderBase {
   }
 
   @Test
-  public void testCodingDirectBuffer_10x4_erasure_of_d2_d4() {
-    prepare(null, 10, 4, new int[] {2, 4});
+  public void testCodingDirectBuffer_10x4_erasure_of_d2_d4_p0() {
+    prepare(null, 10, 4, new int[] {2, 4}, new int[] {0});
     testCoding(true);
   }
 
   @Test
-  public void testCodingDirectBuffer_10x4_erasing_d0_d1() {
-    prepare(null, 10, 4, new int[] {0, 1});
+  public void testCodingDirectBuffer_10x4_erasing_d0_d1_p0_p1() {
+    prepare(null, 10, 4, new int[] {0, 1}, new int[] {0, 1});
     testCoding(true);
   }
 
   @Test
-  public void testCodingNoDirectBuffer_3x3_erasing_d0() {
-    prepare(null, 3, 3, new int[] {0});
+  public void testCodingNoDirectBuffer_3x3_erasing_d0_p0() {
+    prepare(null, 3, 3, new int[] {0}, new int[] {0});
     testCoding(false);
   }
 
   @Test
-  public void testCodingDirectBuffer_3x3_erasing_d0() {
-    prepare(null, 3, 3, new int[] {0});
+  public void testCodingDirectBuffer_3x3_erasing_d0_p0() {
+    prepare(null, 3, 3, new int[] {0}, new int[] {0});
     testCoding(true);
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
index 7ba320a..8543c4d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java
@@ -52,7 +52,7 @@ public abstract class TestRawCoderBase extends TestCoderBase {
     encoder.encode(dataChunks, parityChunks);
 
     // Backup and erase some chunks
-    ECChunk[] backupChunks = backupAndEraseChunks(clonedDataChunks);
+    ECChunk[] backupChunks = backupAndEraseChunks(clonedDataChunks, parityChunks);
 
     // Decode
     ECChunk[] inputChunks = prepareInputChunksForDecoding(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4dc91e58/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
index 62ce4fb..b8912a9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestXORRawCoder.java
@@ -36,7 +36,7 @@ public class TestXORRawCoder extends TestRawCoderBase {
 
   @Test
   public void testCodingNoDirectBuffer_erasing_d0() {
-    prepare(null, 10, 1, new int[] {0});
+    prepare(null, 10, 1, new int[] {0}, new int[0]);
 
     /**
      * Doing twice to test if the coders can be repeatedly reused. This matters
@@ -47,8 +47,15 @@ public class TestXORRawCoder extends TestRawCoderBase {
   }
 
   @Test
+  public void testCodingDirectBuffer_erasing_p0() {
+    prepare(null, 10, 1, new int[0], new int[] {0});
+    testCoding(true);
+    testCoding(true);
+  }
+
+  @Test
   public void testCodingBothBuffers_erasing_d5() {
-    prepare(null, 10, 1, new int[]{5});
+    prepare(null, 10, 1, new int[]{5}, new int[0]);
 
     /**
      * Doing in mixed buffer usage model to test if the coders can be repeatedly


Mime
View raw message