hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From z..@apache.org
Subject [43/50] [abbrv] hadoop git commit: HADOOP-12013 Generate fixed data to perform erasure coder test. Contributed by Kai Zheng
Date Tue, 26 May 2015 19:35:13 GMT
HADOOP-12013 Generate fixed data to perform erasure coder test. Contributed by Kai Zheng


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ae32abd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ae32abd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ae32abd

Branch: refs/heads/HDFS-7285
Commit: 4ae32abdf474f419d65464580f30c678ffff8182
Parents: 3d734df
Author: Kai Zheng <kai.zheng@intel.com>
Authored: Fri May 22 23:47:52 2015 +0800
Committer: Zhe Zhang <zhezhang@cloudera.com>
Committed: Tue May 26 12:07:14 2015 -0700

----------------------------------------------------------------------
 .../hadoop-common/CHANGES-HDFS-EC-7285.txt      |  2 +
 .../hadoop/io/erasurecode/TestCoderBase.java    | 87 +++++++++++++++++++-
 .../io/erasurecode/rawcoder/TestRSRawCoder.java |  8 +-
 .../rawcoder/TestRSRawCoderBase.java            | 21 +++--
 4 files changed, 110 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae32abd/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index c799b4f..531b8d5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -54,3 +54,5 @@
 
     HADOOP-11938. Enhance ByteBuffer version encode/decode API of raw erasure 
     coder. (Kai Zheng via Zhe Zhang)
+
+    HADOOP-12013. Generate fixed data to perform erasure coder test. (Kai Zheng)
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae32abd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
index cc3617c..3686695 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java
@@ -52,6 +52,12 @@ public abstract class TestCoderBase {
   // may go to different coding implementations.
   protected boolean usingDirectBuffer = true;
 
+  protected boolean usingFixedData = true;
+  // Using this the generated data can be repeatable across multiple calls to
+  // encode(), in order for troubleshooting.
+  private static int FIXED_DATA_GENERATOR = 0;
+  protected byte[][] fixedData;
+
   protected int getChunkSize() {
     return chunkSize;
   }
@@ -63,13 +69,17 @@ public abstract class TestCoderBase {
 
   /**
    * Prepare before running the case.
+   * @param conf
    * @param numDataUnits
    * @param numParityUnits
    * @param erasedDataIndexes
+   * @param erasedParityIndexes
+   * @param usingFixedData Using fixed or pre-generated data to test instead of
+   *                       generating data
    */
   protected void prepare(Configuration conf, int numDataUnits,
                          int numParityUnits, int[] erasedDataIndexes,
-                         int[] erasedParityIndexes) {
+                         int[] erasedParityIndexes, boolean usingFixedData) {
     this.conf = conf;
     this.numDataUnits = numDataUnits;
     this.numParityUnits = numParityUnits;
@@ -77,6 +87,38 @@ public abstract class TestCoderBase {
         erasedDataIndexes : new int[] {0};
     this.erasedParityIndexes = erasedParityIndexes != null ?
         erasedParityIndexes : new int[] {0};
+    this.usingFixedData = usingFixedData;
+    if (usingFixedData) {
+      prepareFixedData();
+    }
+  }
+
+  /**
+   * Prepare before running the case.
+   * @param conf
+   * @param numDataUnits
+   * @param numParityUnits
+   * @param erasedDataIndexes
+   * @param erasedParityIndexes
+   */
+  protected void prepare(Configuration conf, int numDataUnits,
+                         int numParityUnits, int[] erasedDataIndexes,
+                         int[] erasedParityIndexes) {
+    prepare(conf, numDataUnits, numParityUnits, erasedDataIndexes,
+        erasedParityIndexes, false);
+  }
+
+  /**
+   * Prepare before running the case.
+   * @param numDataUnits
+   * @param numParityUnits
+   * @param erasedDataIndexes
+   * @param erasedParityIndexes
+   */
+  protected void prepare(int numDataUnits, int numParityUnits,
+                         int[] erasedDataIndexes, int[] erasedParityIndexes) {
+    prepare(null, numDataUnits, numParityUnits, erasedDataIndexes,
+        erasedParityIndexes, false);
   }
 
   /**
@@ -278,6 +320,29 @@ public abstract class TestCoderBase {
    * @return
    */
   protected ECChunk[] prepareDataChunksForEncoding() {
+    if (usingFixedData) {
+      ECChunk[] chunks = new ECChunk[numDataUnits];
+      for (int i = 0; i < chunks.length; i++) {
+        chunks[i] = makeChunkUsingData(fixedData[i]);
+      }
+      return chunks;
+    }
+
+    return generateDataChunks();
+  }
+
+  private ECChunk makeChunkUsingData(byte[] data) {
+    ECChunk chunk = allocateOutputChunk();
+    ByteBuffer buffer = chunk.getBuffer();
+    int pos = buffer.position();
+    buffer.put(data, 0, chunkSize);
+    buffer.flip();
+    buffer.position(pos);
+
+    return chunk;
+  }
+
+  private ECChunk[] generateDataChunks() {
     ECChunk[] chunks = new ECChunk[numDataUnits];
     for (int i = 0; i < chunks.length; i++) {
       chunks[i] = generateDataChunk();
@@ -286,6 +351,15 @@ public abstract class TestCoderBase {
     return chunks;
   }
 
+  private void prepareFixedData() {
+    // We may load test data from a resource, or just generate randomly.
+    // The generated data will be used across subsequent encode/decode calls.
+    this.fixedData = new byte[numDataUnits][];
+    for (int i = 0; i < numDataUnits; i++) {
+      fixedData[i] = generateFixedData(baseChunkSize * 2);
+    }
+  }
+
   /**
    * Generate data chunk by making random data.
    * @return
@@ -319,6 +393,17 @@ public abstract class TestCoderBase {
     return buffer;
   }
 
+  protected byte[] generateFixedData(int len) {
+    byte[] buffer = new byte[len];
+    for (int i = 0; i < buffer.length; i++) {
+      buffer[i] = (byte) FIXED_DATA_GENERATOR++;
+      if (FIXED_DATA_GENERATOR == 256) {
+        FIXED_DATA_GENERATOR = 0;
+      }
+    }
+    return buffer;
+  }
+
   /**
    * Prepare parity chunks for encoding, each chunk for each parity unit.
    * @return

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae32abd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
index 02b9eea..80ec04d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoder.java
@@ -80,7 +80,13 @@ public class TestRSRawCoder extends TestRSRawCoderBase {
 
   @Test
   public void testCodingDirectBuffer_10x4_erasure_of_d2_d4_p0() {
-    prepare(null, 10, 4, new int[] {2, 4}, new int[] {0});
+    prepare(null, 10, 4, new int[]{2, 4}, new int[]{0});
+    testCoding(true);
+  }
+
+  @Test
+  public void testCodingDirectBuffer_usingFixedData_10x4_erasure_of_d2_d4_p0() {
+    prepare(null, 10, 4, new int[] {2, 4}, new int[] {0}, true);
     testCoding(true);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ae32abd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
index c06aded..efde332 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRSRawCoderBase.java
@@ -17,12 +17,7 @@
  */
 package org.apache.hadoop.io.erasurecode.rawcoder;
 
-import org.apache.hadoop.io.erasurecode.ECChunk;
 import org.apache.hadoop.io.erasurecode.rawcoder.util.RSUtil;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.nio.ByteBuffer;
 
 /**
  * Test base for raw Reed-solomon coders.
@@ -32,6 +27,8 @@ public abstract class TestRSRawCoderBase extends TestRawCoderBase {
   private static int symbolSize = 0;
   private static int symbolMax = 0;
 
+  private static int RS_FIXED_DATA_GENERATOR = 0;
+
   static {
     symbolSize = (int) Math.round(Math.log(
         RSUtil.GF.getFieldSize()) / Math.log(2));
@@ -41,9 +38,21 @@ public abstract class TestRSRawCoderBase extends TestRawCoderBase {
   @Override
   protected byte[] generateData(int len) {
     byte[] buffer = new byte[len];
-    for (int i = 0; i < len; i++) {
+    for (int i = 0; i < buffer.length; i++) {
       buffer[i] = (byte) RAND.nextInt(symbolMax);
     }
     return buffer;
   }
+
+  @Override
+  protected byte[] generateFixedData(int len) {
+    byte[] buffer = new byte[len];
+    for (int i = 0; i < buffer.length; i++) {
+      buffer[i] = (byte) RS_FIXED_DATA_GENERATOR++;
+      if (RS_FIXED_DATA_GENERATOR == symbolMax) {
+        RS_FIXED_DATA_GENERATOR = 0;
+      }
+    }
+    return buffer;
+  }
 }


Mime
View raw message