Return-Path: X-Original-To: apmail-hadoop-common-commits-archive@www.apache.org Delivered-To: apmail-hadoop-common-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 91B6417DF6 for ; Mon, 6 Apr 2015 17:39:21 +0000 (UTC) Received: (qmail 73849 invoked by uid 500); 6 Apr 2015 17:38:19 -0000 Delivered-To: apmail-hadoop-common-commits-archive@hadoop.apache.org Received: (qmail 73583 invoked by uid 500); 6 Apr 2015 17:38:18 -0000 Mailing-List: contact common-commits-help@hadoop.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: common-dev@hadoop.apache.org Delivered-To: mailing list common-commits@hadoop.apache.org Received: (qmail 71974 invoked by uid 99); 6 Apr 2015 17:38:15 -0000 Received: from git1-us-west.apache.org (HELO git1-us-west.apache.org) (140.211.11.23) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 06 Apr 2015 17:38:15 +0000 Received: by git1-us-west.apache.org (ASF Mail Server at git1-us-west.apache.org, from userid 33) id 87989E2F1A; Mon, 6 Apr 2015 17:38:01 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: zhz@apache.org To: common-commits@hadoop.apache.org Date: Mon, 06 Apr 2015 17:38:34 -0000 Message-Id: <8adb60f4ab6a434bb7b06a15a217ae62@git.apache.org> In-Reply-To: <374e05a227944c769d59d36f99a37f26@git.apache.org> References: <374e05a227944c769d59d36f99a37f26@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [34/51] [abbrv] hadoop git commit: HADOOP-11706 Refine a little bit erasure coder API HADOOP-11706 Refine a little bit erasure coder API Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37b917d1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37b917d1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37b917d1 Branch: refs/heads/HDFS-7285 Commit: 37b917d14968792aff4f6cead24c7d2ed4e97a91 Parents: ed82e68 Author: Kai Zheng Authored: Wed Mar 18 19:21:37 2015 +0800 Committer: Zhe Zhang Committed: Mon Apr 6 10:21:00 2015 -0700 ---------------------------------------------------------------------- .../io/erasurecode/coder/ErasureCoder.java | 4 +++- .../erasurecode/rawcoder/RawErasureCoder.java | 4 +++- .../hadoop/io/erasurecode/TestCoderBase.java | 17 +++++++++++++--- .../erasurecode/coder/TestErasureCoderBase.java | 21 +++----------------- .../erasurecode/rawcoder/TestJRSRawCoder.java | 12 +++++------ .../erasurecode/rawcoder/TestRawCoderBase.java | 2 ++ 6 files changed, 31 insertions(+), 29 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java index 68875c0..c5922f3 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/coder/ErasureCoder.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.io.erasurecode.coder; +import org.apache.hadoop.conf.Configurable; + /** * An erasure coder to perform encoding or decoding given a group. Generally it * involves calculating necessary internal steps according to codec logic. For @@ -31,7 +33,7 @@ package org.apache.hadoop.io.erasurecode.coder; * of multiple coding steps. * */ -public interface ErasureCoder { +public interface ErasureCoder extends Configurable { /** * Initialize with the important parameters for the code. http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java index 91a9abf..9af5b6c 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/RawErasureCoder.java @@ -17,6 +17,8 @@ */ package org.apache.hadoop.io.erasurecode.rawcoder; +import org.apache.hadoop.conf.Configurable; + /** * RawErasureCoder is a common interface for {@link RawErasureEncoder} and * {@link RawErasureDecoder} as both encoder and decoder share some properties. @@ -31,7 +33,7 @@ package org.apache.hadoop.io.erasurecode.rawcoder; * low level constructs, since it only takes care of the math calculation with * a group of byte buffers. */ -public interface RawErasureCoder { +public interface RawErasureCoder extends Configurable { /** * Initialize with the important parameters for the code. http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java index 194413a..22fd98d 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/TestCoderBase.java @@ -17,11 +17,12 @@ */ package org.apache.hadoop.io.erasurecode; +import org.apache.hadoop.conf.Configuration; + import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Random; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertTrue; /** @@ -31,6 +32,7 @@ import static org.junit.Assert.assertTrue; public abstract class TestCoderBase { protected static Random RAND = new Random(); + private Configuration conf; protected int numDataUnits; protected int numParityUnits; protected int chunkSize = 16 * 1024; @@ -49,8 +51,9 @@ public abstract class TestCoderBase { * @param numParityUnits * @param erasedIndexes */ - protected void prepare(int numDataUnits, int numParityUnits, - int[] erasedIndexes) { + protected void prepare(Configuration conf, int numDataUnits, + int numParityUnits, int[] erasedIndexes) { + this.conf = conf; this.numDataUnits = numDataUnits; this.numParityUnits = numParityUnits; this.erasedDataIndexes = erasedIndexes != null ? @@ -58,6 +61,14 @@ public abstract class TestCoderBase { } /** + * Get the conf the test. + * @return configuration + */ + protected Configuration getConf() { + return this.conf; + } + + /** * Compare and verify if erased chunks are equal to recovered chunks * @param erasedChunks * @param recoveredChunks http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java index d911db9..b963a59 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/coder/TestErasureCoderBase.java @@ -17,10 +17,9 @@ */ package org.apache.hadoop.io.erasurecode.coder; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.erasurecode.ECBlock; -import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.ECBlockGroup; +import org.apache.hadoop.io.erasurecode.ECChunk; import org.apache.hadoop.io.erasurecode.TestCoderBase; /** @@ -30,7 +29,6 @@ public abstract class TestErasureCoderBase extends TestCoderBase { protected Class encoderClass; protected Class decoderClass; - private Configuration conf; protected int numChunksInBlock = 16; /** @@ -48,19 +46,6 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } /** - * Prepare before running the case. - * @param conf - * @param numDataUnits - * @param numParityUnits - * @param erasedIndexes - */ - protected void prepare(Configuration conf, int numDataUnits, - int numParityUnits, int[] erasedIndexes) { - this.conf = conf; - super.prepare(numDataUnits, numParityUnits, erasedIndexes); - } - - /** * Generating source data, encoding, recovering and then verifying. * RawErasureCoder mainly uses ECChunk to pass input and output data buffers, * it supports two kinds of ByteBuffers, one is array backed, the other is @@ -162,7 +147,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } encoder.initialize(numDataUnits, numParityUnits, chunkSize); - ((AbstractErasureCoder)encoder).setConf(conf); + encoder.setConf(getConf()); return encoder; } @@ -179,7 +164,7 @@ public abstract class TestErasureCoderBase extends TestCoderBase { } decoder.initialize(numDataUnits, numParityUnits, chunkSize); - ((AbstractErasureCoder)decoder).setConf(conf); + decoder.setConf(getConf()); return decoder; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java index e54f647..39e5deb 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestJRSRawCoder.java @@ -46,37 +46,37 @@ public class TestJRSRawCoder extends TestRawCoderBase { @Test public void testCodingNoDirectBuffer_10x4() { - prepare(10, 4, null); + prepare(null, 10, 4, null); testCoding(false); } @Test public void testCodingDirectBuffer_10x4() { - prepare(10, 4, null); + prepare(null, 10, 4, null); testCoding(true); } @Test public void testCodingDirectBuffer_10x4_erasure_of_2_4() { - prepare(10, 4, new int[] {2, 4}); + prepare(null, 10, 4, new int[] {2, 4}); testCoding(true); } @Test public void testCodingDirectBuffer_10x4_erasing_all() { - prepare(10, 4, new int[] {0, 1, 2, 3}); + prepare(null, 10, 4, new int[] {0, 1, 2, 3}); testCoding(true); } @Test public void testCodingNoDirectBuffer_3x3() { - prepare(3, 3, null); + prepare(null, 3, 3, null); testCoding(false); } @Test public void testCodingDirectBuffer_3x3() { - prepare(3, 3, null); + prepare(null, 3, 3, null); testCoding(true); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/37b917d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java index 890f632..b036eed 100644 --- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java +++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/erasurecode/rawcoder/TestRawCoderBase.java @@ -86,6 +86,7 @@ public abstract class TestRawCoderBase extends TestCoderBase { } encoder.initialize(numDataUnits, numParityUnits, chunkSize); + encoder.setConf(getConf()); return encoder; } @@ -102,6 +103,7 @@ public abstract class TestRawCoderBase extends TestCoderBase { } decoder.initialize(numDataUnits, numParityUnits, chunkSize); + decoder.setConf(getConf()); return decoder; }