From hcatalog-commits-return-1028-apmail-incubator-hcatalog-commits-archive=incubator.apache.org@incubator.apache.org Fri Aug 31 14:47:36 2012 Return-Path: X-Original-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-hcatalog-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id ADAB2D4F8 for ; Fri, 31 Aug 2012 14:47:36 +0000 (UTC) Received: (qmail 48407 invoked by uid 500); 31 Aug 2012 14:47:36 -0000 Delivered-To: apmail-incubator-hcatalog-commits-archive@incubator.apache.org Received: (qmail 48360 invoked by uid 500); 31 Aug 2012 14:47:36 -0000 Mailing-List: contact hcatalog-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hcatalog-dev@incubator.apache.org Delivered-To: mailing list hcatalog-commits@incubator.apache.org Received: (qmail 48353 invoked by uid 99); 31 Aug 2012 14:47:36 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 31 Aug 2012 14:47:36 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 31 Aug 2012 14:47:34 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 36D8723889B8; Fri, 31 Aug 2012 14:46:52 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1379472 - in /incubator/hcatalog/trunk: CHANGES.txt hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java Date: Fri, 31 Aug 2012 14:46:52 -0000 To: hcatalog-commits@incubator.apache.org From: travis@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20120831144652.36D8723889B8@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: travis Date: Fri Aug 31 14:46:51 2012 New Revision: 1379472 URL: http://svn.apache.org/viewvc?rev=1379472&view=rev Log: HCATALOG-488 TestHCatStorer should extend HCatBaseTest so it uses junit4 and runs inside an IDE Modified: incubator/hcatalog/trunk/CHANGES.txt incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java Modified: incubator/hcatalog/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1379472&r1=1379471&r2=1379472&view=diff ============================================================================== --- incubator/hcatalog/trunk/CHANGES.txt (original) +++ incubator/hcatalog/trunk/CHANGES.txt Fri Aug 31 14:46:51 2012 @@ -38,6 +38,8 @@ Trunk (unreleased changes) HCAT-427 Document storage-based authorization (lefty via gates) IMPROVEMENTS + HCAT-488 TestHCatStorer should extend HCatBaseTest so it uses junit4 and runs inside an IDE (traviscrawford) + HCAT-486 HCatalog should use checkstyle to enforce coding style (traviscrawford) HCAT-484 HCatalog should use ql.metadata Table and Partition classes (traviscrawford) Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java?rev=1379472&r1=1379471&r2=1379472&view=diff ============================================================================== --- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java (original) +++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/test/java/org/apache/hcatalog/pig/TestHCatStorer.java Fri Aug 31 14:46:51 2012 @@ -22,14 +22,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; -import junit.framework.TestCase; - -import org.apache.hadoop.hive.cli.CliSessionState; -import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CommandNeedRetryException; -import org.apache.hadoop.hive.ql.Driver; -import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hcatalog.HcatTestUtils; +import org.apache.hcatalog.mapreduce.HCatBaseTest; import org.apache.pig.ExecType; import org.apache.pig.PigException; import org.apache.pig.PigServer; @@ -37,63 +32,14 @@ import org.apache.pig.data.DataByteArray import org.apache.pig.data.Tuple; import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.util.LogUtils; +import org.junit.Assert; +import org.junit.Test; -public class TestHCatStorer extends TestCase { - private static final String TEST_DATA_DIR = System.getProperty("user.dir") + - "/build/test/data/" + TestHCatStorer.class.getCanonicalName(); - private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; - private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; - - private Driver driver; +public class TestHCatStorer extends HCatBaseTest { - @Override - protected void setUp() throws Exception { - if (driver == null) { - HiveConf hiveConf = new HiveConf(this.getClass()); - hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); - hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); - hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); - hiveConf.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, TEST_WAREHOUSE_DIR); - driver = new Driver(hiveConf); - SessionState.start(new CliSessionState(hiveConf)); - } - } - -// public void testStoreFuncMap() throws IOException{ -// -// driver.run("drop table junit_unparted"); -// String createTable = "create table junit_unparted(b string,arr_of_maps array>) stored as RCFILE " + -// "tblproperties('hcat.isd'='org.apache.hadoop.hive.hCatalog.rcfile.RCFileInputStorageDriver'," + -// "'hcat.osd'='org.apache.hadoop.hive.hCatalog.rcfile.RCFileOutputStorageDriver') "; -// int retCode = driver.run(createTable).getResponseCode(); -// if(retCode != 0) { -// throw new IOException("Failed to create table."); -// } -// -// MiniCluster.deleteFile(cluster, fileName); -// MiniCluster.createInputFile(cluster, fileName, new String[]{"test\t{([a#haddop,b#pig])}","data\t{([b#hive,a#hcat])}"}); -// -// PigServer server = new PigServer(ExecType.LOCAL); -// server.setBatchOn(); -// server.registerQuery("A = load '"+ fullFileName +"' as (b:chararray,arr_of_maps:bag{mytup:tuple ( mymap:map[ ])});"); -// server.registerQuery("store A into 'default.junit_unparted' using org.apache.hadoop.hive.hCatalog.pig.HCatStorer('','b:chararray,arr_of_maps:bag{mytup:tuple ( mymap:map[ ])}');"); -// server.executeBatch(); -// -// -// -// MiniCluster.deleteFile(cluster, fileName); -// -// driver.run("select * from junit_unparted"); -// ArrayList res = new ArrayList(); -// driver.getResults(res); -// driver.run("drop table junit_unparted"); -// Iterator itr = res.iterator(); -// System.out.println(itr.next()); -// System.out.println(itr.next()); -// assertFalse(itr.hasNext()); -// -// } + private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; + @Test public void testPartColsInData() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); @@ -118,16 +64,17 @@ public class TestHCatStorer extends Test while(itr.hasNext()){ Tuple t = itr.next(); - assertEquals(2, t.size()); - assertEquals(t.get(0), i); - assertEquals(t.get(1), "1"); + Assert.assertEquals(2, t.size()); + Assert.assertEquals(t.get(0), i); + Assert.assertEquals(t.get(1), "1"); i++; } - assertFalse(itr.hasNext()); - assertEquals(11, i); + Assert.assertFalse(itr.hasNext()); + Assert.assertEquals(11, i); } + @Test public void testMultiPartColsInData() throws IOException, CommandNeedRetryException{ driver.run("drop table employee"); @@ -161,15 +108,16 @@ public class TestHCatStorer extends Test driver.run("select * from employee"); ArrayList results = new ArrayList(); driver.getResults(results); - assertEquals(4, results.size()); + Assert.assertEquals(4, results.size()); Collections.sort(results); - assertEquals(inputData[0], results.get(0)); - assertEquals(inputData[1], results.get(1)); - assertEquals(inputData[2], results.get(2)); - assertEquals(inputData[3], results.get(3)); + Assert.assertEquals(inputData[0], results.get(0)); + Assert.assertEquals(inputData[1], results.get(1)); + Assert.assertEquals(inputData[2], results.get(2)); + Assert.assertEquals(inputData[3], results.get(3)); driver.run("drop table employee"); } + @Test public void testStoreInPartiitonedTbl() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); @@ -194,16 +142,17 @@ public class TestHCatStorer extends Test while(itr.hasNext()){ Tuple t = itr.next(); - assertEquals(2, t.size()); - assertEquals(t.get(0), i); - assertEquals(t.get(1), "1"); + Assert.assertEquals(2, t.size()); + Assert.assertEquals(t.get(0), i); + Assert.assertEquals(t.get(1), "1"); i++; } - assertFalse(itr.hasNext()); - assertEquals(11, i); + Assert.assertFalse(itr.hasNext()); + Assert.assertEquals(11, i); } + @Test public void testNoAlias() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_parted"); String createTable = "create table junit_parted(a int, b string) partitioned by (ds string) stored as RCFILE"; @@ -222,12 +171,12 @@ public class TestHCatStorer extends Test } catch(PigException fe){ PigException pe = LogUtils.getPigException(fe); - assertTrue(pe instanceof FrontendException); - assertEquals(PigHCatUtil.PIG_EXCEPTION_CODE, pe.getErrorCode()); - assertTrue(pe.getMessage().contains("Column name for a field is not specified. Please provide the full schema as an argument to HCatStorer.")); + Assert.assertTrue(pe instanceof FrontendException); + Assert.assertEquals(PigHCatUtil.PIG_EXCEPTION_CODE, pe.getErrorCode()); + Assert.assertTrue(pe.getMessage().contains("Column name for a field is not specified. Please provide the full schema as an argument to HCatStorer.")); errCaught = true; } - assertTrue(errCaught); + Assert.assertTrue(errCaught); errCaught = false; try{ server.setBatchOn(); @@ -238,15 +187,16 @@ public class TestHCatStorer extends Test } catch(PigException fe){ PigException pe = LogUtils.getPigException(fe); - assertTrue(pe instanceof FrontendException); - assertEquals(PigHCatUtil.PIG_EXCEPTION_CODE, pe.getErrorCode()); - assertTrue(pe.getMessage().contains("Column names should all be in lowercase. Invalid name found: B")); + Assert.assertTrue(pe instanceof FrontendException); + Assert.assertEquals(PigHCatUtil.PIG_EXCEPTION_CODE, pe.getErrorCode()); + Assert.assertTrue(pe.getMessage().contains("Column names should all be in lowercase. Invalid name found: B")); errCaught = true; } driver.run("drop table junit_parted"); - assertTrue(errCaught); + Assert.assertTrue(errCaught); } + @Test public void testStoreMultiTables() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); @@ -294,13 +244,14 @@ public class TestHCatStorer extends Test Iterator itr = res.iterator(); for(int i = 0; i < LOOP_SIZE*LOOP_SIZE; i++) { - assertEquals( input[i] ,itr.next()); + Assert.assertEquals( input[i] ,itr.next()); } - assertFalse(itr.hasNext()); + Assert.assertFalse(itr.hasNext()); } + @Test public void testStoreWithNoSchema() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); @@ -332,13 +283,14 @@ public class TestHCatStorer extends Test driver.run("drop table junit_unparted"); Iterator itr = res.iterator(); for(int i = 0; i < LOOP_SIZE*LOOP_SIZE; i++) { - assertEquals( input[i] ,itr.next()); + Assert.assertEquals( input[i] ,itr.next()); } - assertFalse(itr.hasNext()); + Assert.assertFalse(itr.hasNext()); } + @Test public void testStoreWithNoCtorArgs() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); @@ -370,13 +322,14 @@ public class TestHCatStorer extends Test driver.run("drop table junit_unparted"); Iterator itr = res.iterator(); for(int i = 0; i < LOOP_SIZE*LOOP_SIZE; i++) { - assertEquals( input[i] ,itr.next()); + Assert.assertEquals( input[i] ,itr.next()); } - assertFalse(itr.hasNext()); + Assert.assertFalse(itr.hasNext()); } + @Test public void testEmptyStore() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); @@ -408,10 +361,11 @@ public class TestHCatStorer extends Test driver.getResults(res); driver.run("drop table junit_unparted"); Iterator itr = res.iterator(); - assertFalse(itr.hasNext()); + Assert.assertFalse(itr.hasNext()); } + @Test public void testBagNStruct() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); String createTable = "create table junit_unparted(b string,a struct, arr_of_struct array, " + @@ -438,12 +392,13 @@ public class TestHCatStorer extends Test driver.getResults(res); driver.run("drop table junit_unparted"); Iterator itr = res.iterator(); - assertEquals("zookeeper\t{\"a1\":2}\t[\"pig\"]\t[{\"s1\":\"pnuts\",\"s2\":\"hdfs\"}]\t[{\"s3\":\"hadoop\"},{\"s3\":\"hcat\"}]", itr.next()); - assertEquals("chubby\t{\"a1\":2}\t[\"sawzall\"]\t[{\"s1\":\"bigtable\",\"s2\":\"gfs\"}]\t[{\"s3\":\"mapreduce\"},{\"s3\":\"hcat\"}]",itr.next()); - assertFalse(itr.hasNext()); + Assert.assertEquals("zookeeper\t{\"a1\":2}\t[\"pig\"]\t[{\"s1\":\"pnuts\",\"s2\":\"hdfs\"}]\t[{\"s3\":\"hadoop\"},{\"s3\":\"hcat\"}]", itr.next()); + Assert.assertEquals("chubby\t{\"a1\":2}\t[\"sawzall\"]\t[{\"s1\":\"bigtable\",\"s2\":\"gfs\"}]\t[{\"s3\":\"mapreduce\"},{\"s3\":\"hcat\"}]",itr.next()); + Assert.assertFalse(itr.hasNext()); } + @Test public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); @@ -473,10 +428,10 @@ public class TestHCatStorer extends Test driver.getResults(res); Iterator itr = res.iterator(); - assertEquals( "0\tNULL\tNULL\tNULL\tNULL\tnull\tnull" ,itr.next()); - assertEquals( "NULL\t4.2\t2.2\t4\tlets hcat\tbinary-data\tnull" ,itr.next()); - assertEquals( "3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tbinary-data\tnull",itr.next()); - assertFalse(itr.hasNext()); + Assert.assertEquals( "0\tNULL\tNULL\tNULL\tNULL\tnull\tnull" ,itr.next()); + Assert.assertEquals( "NULL\t4.2\t2.2\t4\tlets hcat\tbinary-data\tnull" ,itr.next()); + Assert.assertEquals( "3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tbinary-data\tnull",itr.next()); + Assert.assertFalse(itr.hasNext()); server.registerQuery("B = load 'junit_unparted' using "+HCatLoader.class.getName()+";"); Iterator iter = server.openIterator("B"); @@ -487,24 +442,17 @@ public class TestHCatStorer extends Test if(t.get(5) == null){ num5nulls++; }else { - assertTrue(t.get(5) instanceof DataByteArray); + Assert.assertTrue(t.get(5) instanceof DataByteArray); } - assertNull(t.get(6)); + Assert.assertNull(t.get(6)); count++; } - assertEquals(3, count); - assertEquals(1, num5nulls); + Assert.assertEquals(3, count); + Assert.assertEquals(1, num5nulls); driver.run("drop table junit_unparted"); } - @Override - protected void tearDown() throws Exception { - super.tearDown(); - } - - - - + @Test public void testStoreFuncSimple() throws IOException, CommandNeedRetryException{ driver.run("drop table junit_unparted"); @@ -538,14 +486,14 @@ public class TestHCatStorer extends Test for(int i = 1; i <= LOOP_SIZE; i++) { String si = i + ""; for(int j=1;j<=LOOP_SIZE;j++) { - assertEquals( si + "\t"+j,itr.next()); + Assert.assertEquals( si + "\t"+j,itr.next()); } } - assertFalse(itr.hasNext()); + Assert.assertFalse(itr.hasNext()); } - + @Test public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws IOException, CommandNeedRetryException{ driver.run("drop table if exists employee"); @@ -573,15 +521,16 @@ public class TestHCatStorer extends Test driver.run("select * from employee"); ArrayList results = new ArrayList(); driver.getResults(results); - assertEquals(4, results.size()); + Assert.assertEquals(4, results.size()); Collections.sort(results); - assertEquals(inputData[0], results.get(0)); - assertEquals(inputData[1], results.get(1)); - assertEquals(inputData[2], results.get(2)); - assertEquals(inputData[3], results.get(3)); + Assert.assertEquals(inputData[0], results.get(0)); + Assert.assertEquals(inputData[1], results.get(1)); + Assert.assertEquals(inputData[2], results.get(2)); + Assert.assertEquals(inputData[3], results.get(3)); driver.run("drop table employee"); } + @Test public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws IOException, CommandNeedRetryException{ driver.run("drop table if exists employee"); @@ -609,15 +558,16 @@ public class TestHCatStorer extends Test driver.run("select * from employee"); ArrayList results = new ArrayList(); driver.getResults(results); - assertEquals(4, results.size()); + Assert.assertEquals(4, results.size()); Collections.sort(results); - assertEquals(inputData[0], results.get(0)); - assertEquals(inputData[1], results.get(1)); - assertEquals(inputData[2], results.get(2)); - assertEquals(inputData[3], results.get(3)); + Assert.assertEquals(inputData[0], results.get(0)); + Assert.assertEquals(inputData[1], results.get(1)); + Assert.assertEquals(inputData[2], results.get(2)); + Assert.assertEquals(inputData[3], results.get(3)); driver.run("drop table employee"); } + @Test public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws IOException, CommandNeedRetryException{ driver.run("drop table if exists employee"); @@ -642,7 +592,7 @@ public class TestHCatStorer extends Test driver.run("select * from employee"); ArrayList results = new ArrayList(); driver.getResults(results); - assertEquals(0, results.size()); + Assert.assertEquals(0, results.size()); driver.run("drop table employee"); } }