From hcatalog-commits-return-128-apmail-incubator-hcatalog-commits-archive=incubator.apache.org@incubator.apache.org Thu May 5 17:45:59 2011 Return-Path: X-Original-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4034A269F for ; Thu, 5 May 2011 17:45:59 +0000 (UTC) Received: (qmail 36714 invoked by uid 500); 5 May 2011 17:45:59 -0000 Delivered-To: apmail-incubator-hcatalog-commits-archive@incubator.apache.org Received: (qmail 36695 invoked by uid 500); 5 May 2011 17:45:59 -0000 Mailing-List: contact hcatalog-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hcatalog-dev@incubator.apache.org Delivered-To: mailing list hcatalog-commits@incubator.apache.org Received: (qmail 36688 invoked by uid 99); 5 May 2011 17:45:59 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 May 2011 17:45:59 +0000 X-ASF-Spam-Status: No, hits=-1998.9 required=5.0 tests=ALL_TRUSTED,FRT_ADOBE2 X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 05 May 2011 17:45:57 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 3BD2B238896F; Thu, 5 May 2011 17:45:37 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1099891 - in /incubator/hcatalog/trunk/src: java/org/apache/hcatalog/common/HCatUtil.java test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java Date: Thu, 05 May 2011 17:45:37 -0000 To: hcatalog-commits@incubator.apache.org From: hashutosh@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110505174537.3BD2B238896F@eris.apache.org> Author: hashutosh Date: Thu May 5 17:45:36 2011 New Revision: 1099891 URL: http://svn.apache.org/viewvc?rev=1099891&view=rev Log: HCAT-18: Refactor TestRCFileInputStorageDriver Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java?rev=1099891&r1=1099890&r2=1099891&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java (original) +++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/common/HCatUtil.java Thu May 5 17:45:36 2011 @@ -100,6 +100,16 @@ public class HCatUtil { return bytes; } + public static List getHCatFieldSchemaList(FieldSchema ... fields) throws HCatException { + List result = new ArrayList(fields.length); + + for(FieldSchema f : fields) { + result.add(HCatSchemaUtils.getHCatFieldSchema(f)); + } + + return result; + } + public static List getHCatFieldSchemaList(List fields) throws HCatException { if(fields == null) { return null; Modified: incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java?rev=1099891&r1=1099890&r2=1099891&view=diff ============================================================================== --- incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java (original) +++ incubator/hcatalog/trunk/src/test/org/apache/hcatalog/rcfile/TestRCFileInputStorageDriver.java Thu May 5 17:45:36 2011 @@ -15,12 +15,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.hcatalog.rcfile;import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; +package org.apache.hcatalog.rcfile; + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.*; import junit.framework.Assert; import junit.framework.TestCase; @@ -49,58 +48,54 @@ import org.apache.hcatalog.data.DefaultH import org.apache.hcatalog.data.HCatRecord; import org.apache.hcatalog.data.schema.HCatSchema; import org.apache.hcatalog.rcfile.RCFileInputDriver; -; - -public class TestRCFileInputStorageDriver extends TestCase{ - - private static Configuration conf = new Configuration(); - - private static Path file; - private static FileSystem fs; - - static { - - try { - fs = FileSystem.getLocal(conf); - Path dir = new Path(System.getProperty("test.data.dir", ".") + "/mapred"); - file = new Path(dir, "test_rcfile"); - fs.delete(dir, true); - } catch (Exception e) { - } - } - public void testConvertValueToTuple() throws IOException,InterruptedException{ - fs.delete(file, true); +public class TestRCFileInputStorageDriver extends TestCase{ + private static final Configuration conf = new Configuration(); + private static final Path dir = new Path(System.getProperty("test.data.dir", ".") + "/mapred"); + private static final Path file = new Path(dir, "test_rcfile"); + // Generate sample records to compare against + private byte[][][] getRecords() throws UnsupportedEncodingException { byte[][] record_1 = {"123".getBytes("UTF-8"), "456".getBytes("UTF-8"), "789".getBytes("UTF-8"), "1000".getBytes("UTF-8"), - "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"), + "5.3".getBytes("UTF-8"), "hcatalog and hadoop".getBytes("UTF-8"), new byte[0], "\\N".getBytes("UTF-8")}; byte[][] record_2 = {"100".getBytes("UTF-8"), "200".getBytes("UTF-8"), "123".getBytes("UTF-8"), "1000".getBytes("UTF-8"), - "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"), + "5.3".getBytes("UTF-8"), "hcatalog and hadoop".getBytes("UTF-8"), new byte[0], "\\N".getBytes("UTF-8")}; + return new byte[][][]{record_1, record_2}; + } + // Write sample records to file for individual tests + private BytesRefArrayWritable[] initTestEnvironment() throws IOException { + FileSystem fs = FileSystem.getLocal(conf); + fs.delete(file, true); + + byte [][][] records = getRecords(); RCFileOutputFormat.setColumnNumber(conf, 8); - RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, - new DefaultCodec()); - BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length); - for (int i = 0; i < record_1.length; i++) { - BytesRefWritable cu = new BytesRefWritable(record_1[i], 0, - record_1[i].length); + RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, new DefaultCodec()); + + BytesRefArrayWritable bytes = writeBytesToFile(records[0], writer); + BytesRefArrayWritable bytes2 = writeBytesToFile(records[1], writer); + + writer.close(); + return new BytesRefArrayWritable[]{bytes,bytes2}; + } + + private BytesRefArrayWritable writeBytesToFile(byte[][] record, RCFile.Writer writer) throws IOException { + BytesRefArrayWritable bytes = new BytesRefArrayWritable(record.length); + for (int i = 0; i < record.length; i++) { + BytesRefWritable cu = new BytesRefWritable(record[i], 0, record[i].length); bytes.set(i, cu); } writer.append(bytes); - BytesRefArrayWritable bytes2 = new BytesRefArrayWritable(record_2.length); - for (int i = 0; i < record_2.length; i++) { - BytesRefWritable cu = new BytesRefWritable(record_2[i], 0, - record_2[i].length); - bytes2.set(i, cu); - } - writer.append(bytes2); - writer.close(); - BytesRefArrayWritable[] bytesArr = new BytesRefArrayWritable[]{bytes,bytes2}; + return bytes; + } + + public void testConvertValueToTuple() throws IOException,InterruptedException{ + BytesRefArrayWritable[] bytesArr = initTestEnvironment(); HCatSchema schema = buildHiveSchema(); RCFileInputDriver sd = new RCFileInputDriver(); @@ -127,36 +122,7 @@ public class TestRCFileInputStorageDrive } public void testPruning() throws IOException,InterruptedException{ - fs.delete(file, true); - - byte[][] record_1 = {"123".getBytes("UTF-8"), "456".getBytes("UTF-8"), - "789".getBytes("UTF-8"), "1000".getBytes("UTF-8"), - "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"), - new byte[0], "\\N".getBytes("UTF-8")}; - byte[][] record_2 = {"100".getBytes("UTF-8"), "200".getBytes("UTF-8"), - "123".getBytes("UTF-8"), "1000".getBytes("UTF-8"), - "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"), - new byte[0], "\\N".getBytes("UTF-8")}; - - RCFileOutputFormat.setColumnNumber(conf, 8); - RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, - new DefaultCodec()); - BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length); - for (int i = 0; i < record_1.length; i++) { - BytesRefWritable cu = new BytesRefWritable(record_1[i], 0, - record_1[i].length); - bytes.set(i, cu); - } - writer.append(bytes); - BytesRefArrayWritable bytes2 = new BytesRefArrayWritable(record_2.length); - for (int i = 0; i < record_2.length; i++) { - BytesRefWritable cu = new BytesRefWritable(record_2[i], 0, - record_2[i].length); - bytes2.set(i, cu); - } - writer.append(bytes2); - writer.close(); - BytesRefArrayWritable[] bytesArr = new BytesRefArrayWritable[]{bytes,bytes2}; + BytesRefArrayWritable[] bytesArr = initTestEnvironment(); RCFileInputDriver sd = new RCFileInputDriver(); JobContext jc = new JobContext(conf, new JobID()); @@ -185,36 +151,7 @@ public class TestRCFileInputStorageDrive } public void testReorderdCols() throws IOException,InterruptedException{ - fs.delete(file, true); - - byte[][] record_1 = {"123".getBytes("UTF-8"), "456".getBytes("UTF-8"), - "789".getBytes("UTF-8"), "1000".getBytes("UTF-8"), - "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"), - new byte[0], "\\N".getBytes("UTF-8")}; - byte[][] record_2 = {"100".getBytes("UTF-8"), "200".getBytes("UTF-8"), - "123".getBytes("UTF-8"), "1000".getBytes("UTF-8"), - "5.3".getBytes("UTF-8"), "howl and hadoop".getBytes("UTF-8"), - new byte[0], "\\N".getBytes("UTF-8")}; - - RCFileOutputFormat.setColumnNumber(conf, 8); - RCFile.Writer writer = new RCFile.Writer(fs, conf, file, null, - new DefaultCodec()); - BytesRefArrayWritable bytes = new BytesRefArrayWritable(record_1.length); - for (int i = 0; i < record_1.length; i++) { - BytesRefWritable cu = new BytesRefWritable(record_1[i], 0, - record_1[i].length); - bytes.set(i, cu); - } - writer.append(bytes); - BytesRefArrayWritable bytes2 = new BytesRefArrayWritable(record_2.length); - for (int i = 0; i < record_2.length; i++) { - BytesRefWritable cu = new BytesRefWritable(record_2[i], 0, - record_2[i].length); - bytes2.set(i, cu); - } - writer.append(bytes2); - writer.close(); - BytesRefArrayWritable[] bytesArr = new BytesRefArrayWritable[]{bytes,bytes2}; + BytesRefArrayWritable[] bytesArr = initTestEnvironment(); RCFileInputDriver sd = new RCFileInputDriver(); JobContext jc = new JobContext(conf, new JobID()); @@ -244,122 +181,101 @@ public class TestRCFileInputStorageDrive } assertFalse(rr.nextKeyValue()); } - private HCatRecord[] getExpectedRecords(){ - List rec_1 = new ArrayList(8); - rec_1.add(new Byte("123")); - rec_1.add(new Short("456")); - rec_1.add( new Integer(789)); - rec_1.add( new Long(1000L)); - rec_1.add( new Double(5.3D)); - rec_1.add( new String("howl and hadoop")); - rec_1.add( null); - rec_1.add( null); + Collections.addAll(rec_1, new Byte("123"), + new Short("456"), + new Integer(789), + new Long(1000L), + new Double(5.3D), + new String("hcatalog and hadoop"), + null, + null); HCatRecord tup_1 = new DefaultHCatRecord(rec_1); List rec_2 = new ArrayList(8); - rec_2.add( new Byte("100")); - rec_2.add( new Short("200")); - rec_2.add( new Integer(123)); - rec_2.add( new Long(1000L)); - rec_2.add( new Double(5.3D)); - rec_2.add( new String("howl and hadoop")); - rec_2.add( null); - rec_2.add( null); + Collections.addAll(rec_2, new Byte("100"), + new Short("200"), + new Integer(123), + new Long(1000L), + new Double(5.3D), + new String("hcatalog and hadoop"), + null, + null); HCatRecord tup_2 = new DefaultHCatRecord(rec_2); return new HCatRecord[]{tup_1,tup_2}; - } private HCatRecord[] getPrunedRecords(){ - List rec_1 = new ArrayList(8); - rec_1.add(new Byte("123")); - rec_1.add( new Integer(789)); - rec_1.add( new Double(5.3D)); - rec_1.add( new String("howl and hadoop")); - rec_1.add( null); + Collections.addAll(rec_1, new Byte("123"), + new Integer(789), + new Double(5.3D), + new String("hcatalog and hadoop"), + null); HCatRecord tup_1 = new DefaultHCatRecord(rec_1); List rec_2 = new ArrayList(8); - rec_2.add( new Byte("100")); - rec_2.add( new Integer(123)); - rec_2.add( new Double(5.3D)); - rec_2.add( new String("howl and hadoop")); - rec_2.add( null); + Collections.addAll(rec_2, new Byte("100"), + new Integer(123), + new Double(5.3D), + new String("hcatalog and hadoop"), + null); HCatRecord tup_2 = new DefaultHCatRecord(rec_2); return new HCatRecord[]{tup_1,tup_2}; - } private HCatSchema buildHiveSchema() throws HCatException{ - - List fields = new ArrayList(8); - fields.add(new FieldSchema("atinyint", "tinyint", "")); - fields.add(new FieldSchema("asmallint", "smallint", "")); - fields.add(new FieldSchema("aint", "int", "")); - fields.add(new FieldSchema("along", "bigint", "")); - fields.add(new FieldSchema("adouble", "double", "")); - fields.add(new FieldSchema("astring", "string", "")); - fields.add(new FieldSchema("anullint", "int", "")); - fields.add(new FieldSchema("anullstring", "string", "")); - - return new HCatSchema(HCatUtil.getHCatFieldSchemaList(fields)); + return new HCatSchema(HCatUtil.getHCatFieldSchemaList(new FieldSchema("atinyint", "tinyint", ""), + new FieldSchema("asmallint", "smallint", ""), + new FieldSchema("aint", "int", ""), + new FieldSchema("along", "bigint", ""), + new FieldSchema("adouble", "double", ""), + new FieldSchema("astring", "string", ""), + new FieldSchema("anullint", "int", ""), + new FieldSchema("anullstring", "string", ""))); } private HCatSchema buildPrunedSchema() throws HCatException{ - - List fields = new ArrayList(5); - fields.add(new FieldSchema("atinyint", "tinyint", "")); - fields.add(new FieldSchema("aint", "int", "")); - fields.add(new FieldSchema("adouble", "double", "")); - fields.add(new FieldSchema("astring", "string", "")); - fields.add(new FieldSchema("anullint", "int", "")); - - return new HCatSchema(HCatUtil.getHCatFieldSchemaList(fields)); + return new HCatSchema(HCatUtil.getHCatFieldSchemaList(new FieldSchema("atinyint", "tinyint", ""), + new FieldSchema("aint", "int", ""), + new FieldSchema("adouble", "double", ""), + new FieldSchema("astring", "string", ""), + new FieldSchema("anullint", "int", ""))); } private HCatSchema buildReorderedSchema() throws HCatException{ - - List fields = new ArrayList(7); - - fields.add(new FieldSchema("aint", "int", "")); - fields.add(new FieldSchema("part1", "string", "")); - fields.add(new FieldSchema("adouble", "double", "")); - fields.add(new FieldSchema("newCol", "tinyint", "")); - fields.add(new FieldSchema("astring", "string", "")); - fields.add(new FieldSchema("atinyint", "tinyint", "")); - fields.add(new FieldSchema("anullint", "int", "")); - - - return new HCatSchema(HCatUtil.getHCatFieldSchemaList(fields)); + return new HCatSchema(HCatUtil.getHCatFieldSchemaList(new FieldSchema("aint", "int", ""), + new FieldSchema("part1", "string", ""), + new FieldSchema("adouble", "double", ""), + new FieldSchema("newCol", "tinyint", ""), + new FieldSchema("astring", "string", ""), + new FieldSchema("atinyint", "tinyint", ""), + new FieldSchema("anullint", "int", ""))); } private HCatRecord[] getReorderedCols(){ - List rec_1 = new ArrayList(7); - rec_1.add( new Integer(789)); - rec_1.add( new String("first-part")); - rec_1.add( new Double(5.3D)); - rec_1.add( null); // new column - rec_1.add( new String("howl and hadoop")); - rec_1.add( new Byte("123")); - rec_1.add( null); + Collections.addAll(rec_1, new Integer(789), + new String("first-part"), + new Double(5.3D), + null, // new column + new String("hcatalog and hadoop"), + new Byte("123"), + null); HCatRecord tup_1 = new DefaultHCatRecord(rec_1); List rec_2 = new ArrayList(7); - - rec_2.add( new Integer(123)); - rec_2.add( new String("first-part")); - rec_2.add( new Double(5.3D)); - rec_2.add(null); - rec_2.add( new String("howl and hadoop")); - rec_2.add( new Byte("100")); - rec_2.add( null); + Collections.addAll(rec_2, new Integer(123), + new String("first-part"), + new Double(5.3D), + null, + new String("hcatalog and hadoop"), + new Byte("100"), + null); HCatRecord tup_2 = new DefaultHCatRecord(rec_2); return new HCatRecord[]{tup_1,tup_2};