Return-Path: X-Original-To: apmail-hive-commits-archive@www.apache.org Delivered-To: apmail-hive-commits-archive@www.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 4662317302 for ; Wed, 1 Oct 2014 09:44:38 +0000 (UTC) Received: (qmail 60313 invoked by uid 500); 1 Oct 2014 09:44:38 -0000 Delivered-To: apmail-hive-commits-archive@hive.apache.org Received: (qmail 60271 invoked by uid 500); 1 Oct 2014 09:44:38 -0000 Mailing-List: contact commits-help@hive.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: hive-dev@hive.apache.org Delivered-To: mailing list commits@hive.apache.org Received: (qmail 60260 invoked by uid 99); 1 Oct 2014 09:44:38 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Oct 2014 09:44:38 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 01 Oct 2014 09:44:33 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 7C16823889E1; Wed, 1 Oct 2014 09:44:13 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1628661 - in /hive/branches/branch-0.14: hcatalog/hcatalog-pig-adapter/ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/ ql/src/test/org/apache/hadoop/hive/ql/io/ Date: Wed, 01 Oct 2014 09:44:13 -0000 To: commits@hive.apache.org From: khorgath@apache.org X-Mailer: svnmailer-1.0.9 Message-Id: <20141001094413.7C16823889E1@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: khorgath Date: Wed Oct 1 09:44:12 2014 New Revision: 1628661 URL: http://svn.apache.org/r1628661 Log: HIVE-7420 : Parameterize tests for HCatalog Pig interfaces for testing against all storage formats (David Chen via Sushanth Sowmyan) Added: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestUtil.java Removed: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatLoader.java hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatLoaderComplexSchema.java hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java Modified: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/pom.xml hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java Modified: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/pom.xml URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/pom.xml?rev=1628661&r1=1628660&r2=1628661&view=diff ============================================================================== --- hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/pom.xml (original) +++ hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/pom.xml Wed Oct 1 09:44:12 2014 @@ -53,6 +53,13 @@ tests test + + org.apache.hive + hive-exec + ${project.version} + test-jar + test + Modified: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1628661&r1=1628660&r2=1628661&view=diff ============================================================================== --- hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original) +++ hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Wed Oct 1 09:44:12 2014 @@ -28,10 +28,12 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.Set; import org.apache.commons.io.FileUtils; @@ -42,6 +44,8 @@ import org.apache.hadoop.hive.cli.CliSes import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.StorageFormats; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.serde2.ColumnProjectionUtils; @@ -69,12 +73,16 @@ import org.joda.time.DateTime; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; +@RunWith(Parameterized.class) public class TestHCatLoader { private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoader.class); private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") + @@ -91,9 +99,30 @@ public class TestHCatLoader { private Driver driver; private Map> basicInputData; - protected String storageFormat() { - return "RCFILE tblproperties('hcat.isd'='org.apache.hive.hcatalog.rcfile.RCFileInputDriver'," + - "'hcat.osd'='org.apache.hive.hcatalog.rcfile.RCFileOutputDriver')"; + private static final Map> DISABLED_STORAGE_FORMATS = + new HashMap>() {{ + put(IOConstants.AVRO, new HashSet() {{ + add("testReadDataBasic"); + add("testReadPartitionedBasic"); + add("testProjectionsBasic"); + add("testSchemaLoadPrimitiveTypes"); + }}); + put(IOConstants.PARQUETFILE, new HashSet() {{ + add("testReadDataBasic"); + add("testReadPartitionedBasic"); + add("testProjectionsBasic"); + }}); + }}; + + private String storageFormat; + + @Parameterized.Parameters + public static Collection generateParameters() { + return StorageFormats.names(); + } + + public TestHCatLoader(String storageFormat) { + this.storageFormat = storageFormat; } private void dropTable(String tablename) throws IOException, CommandNeedRetryException { @@ -105,7 +134,7 @@ public class TestHCatLoader { } private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException { - createTable(tablename, schema, partitionedBy, driver, storageFormat()); + createTable(tablename, schema, partitionedBy, driver, storageFormat); } static void createTable(String tablename, String schema, String partitionedBy, Driver driver, String storageFormat) @@ -209,17 +238,18 @@ public class TestHCatLoader { server.registerQuery("D = load '" + COMPLEX_FILE_NAME + "' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});", ++i); server.registerQuery("store D into '" + COMPLEX_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();", ++i); server.executeBatch(); - } @After public void tearDown() throws Exception { try { - dropTable(BASIC_TABLE); - dropTable(COMPLEX_TABLE); - dropTable(PARTITIONED_TABLE); - dropTable(SPECIFIC_SIZE_TABLE); - dropTable(AllTypesTable.ALL_PRIMITIVE_TYPES_TABLE); + if (driver != null) { + dropTable(BASIC_TABLE); + dropTable(COMPLEX_TABLE); + dropTable(PARTITIONED_TABLE); + dropTable(SPECIFIC_SIZE_TABLE); + dropTable(AllTypesTable.ALL_PRIMITIVE_TYPES_TABLE); + } } finally { FileUtils.deleteDirectory(new File(TEST_DATA_DIR)); } @@ -227,6 +257,7 @@ public class TestHCatLoader { @Test public void testSchemaLoadBasic() throws IOException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); PigServer server = new PigServer(ExecType.LOCAL); @@ -241,23 +272,28 @@ public class TestHCatLoader { assertTrue(Xfields.get(1).type == DataType.CHARARRAY); } + /** * Test that we properly translate data types in Hive/HCat table schema into Pig schema */ @Test public void testSchemaLoadPrimitiveTypes() throws IOException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); AllTypesTable.testSchemaLoadPrimitiveTypes(); } + /** * Test that value from Hive table are read properly in Pig */ @Test public void testReadDataPrimitiveTypes() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); AllTypesTable.testReadDataPrimitiveTypes(); } @Test public void testReadDataBasic() throws IOException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); PigServer server = new PigServer(ExecType.LOCAL); server.registerQuery("X = load '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatLoader();"); @@ -279,6 +315,7 @@ public class TestHCatLoader { @Test public void testSchemaLoadComplex() throws IOException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); PigServer server = new PigServer(ExecType.LOCAL); @@ -337,6 +374,7 @@ public class TestHCatLoader { @Test public void testReadPartitionedBasic() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); PigServer server = new PigServer(ExecType.LOCAL); driver.run("select * from " + PARTITIONED_TABLE); @@ -404,6 +442,7 @@ public class TestHCatLoader { @Test public void testProjectionsBasic() throws IOException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); PigServer server = new PigServer(ExecType.LOCAL); @@ -453,6 +492,7 @@ public class TestHCatLoader { @Test public void testColumnarStorePushdown() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); String PIGOUTPUT_DIR = TEST_DATA_DIR+ "/colpushdownop"; String PIG_FILE = "test.pig"; String expectedCols = "0,1"; @@ -486,6 +526,7 @@ public class TestHCatLoader { @Test public void testGetInputBytes() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); File file = new File(TEST_WAREHOUSE_DIR + "/" + SPECIFIC_SIZE_TABLE + "/part-m-00000"); file.deleteOnExit(); RandomAccessFile randomAccessFile = new RandomAccessFile(file, "rw"); @@ -501,6 +542,7 @@ public class TestHCatLoader { @Test public void testConvertBooleanToInt() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); String tbl = "test_convert_boolean_to_int"; String inputFileName = TEST_DATA_DIR + "/testConvertBooleanToInt/data.txt"; File inputDataDir = new File(inputFileName).getParentFile(); @@ -608,22 +650,26 @@ public class TestHCatLoader { Tuple t = XIter.next(); assertEquals(HCatFieldSchema.Type.numPrimitiveTypes(), t.size()); int colPos = 0; - for(Object referenceData : primitiveRows[numTuplesRead]) { - if(referenceData == null) { - assertTrue("rowNum=" + numTuplesRead + " colNum=" + colPos + " Reference data is null; actual " + - t.get(colPos), t.get(colPos) == null); - } - else if(referenceData instanceof java.util.Date) { - assertTrue("rowNum=" + numTuplesRead + " colNum=" + colPos + " Reference data=" + ((java.util.Date)referenceData).getTime() + " actual=" + - ((DateTime)t.get(colPos)).getMillis() + "; types=(" + referenceData.getClass() + "," + t.get(colPos).getClass() + ")", + for (Object referenceData : primitiveRows[numTuplesRead]) { + if (referenceData == null) { + assertTrue("rowNum=" + numTuplesRead + " colNum=" + colPos + + " Reference data is null; actual " + + t.get(colPos), t.get(colPos) == null); + } else if (referenceData instanceof java.util.Date) { + // Note that here we ignore nanos part of Hive Timestamp since nanos are dropped when + // reading Hive from Pig by design. + assertTrue("rowNum=" + numTuplesRead + " colNum=" + colPos + + " Reference data=" + ((java.util.Date)referenceData).getTime() + + " actual=" + ((DateTime)t.get(colPos)).getMillis() + + "; types=(" + referenceData.getClass() + "," + t.get(colPos).getClass() + ")", ((java.util.Date)referenceData).getTime()== ((DateTime)t.get(colPos)).getMillis()); - //note that here we ignore nanos part of Hive Timestamp since nanos are dropped when reading Hive from Pig by design - } - else { - assertTrue("rowNum=" + numTuplesRead + " colNum=" + colPos + " Reference data=" + referenceData + " actual=" + - t.get(colPos) + "; types=(" + referenceData.getClass() + "," + t.get(colPos).getClass() + ")", + } else { + // Doing String comps here as value objects in Hive in Pig are different so equals() + // doesn't work. + assertTrue("rowNum=" + numTuplesRead + " colNum=" + colPos + + " Reference data=" + referenceData + " actual=" + t.get(colPos) + + "; types=(" + referenceData.getClass() + "," + t.get(colPos).getClass() + ") ", referenceData.toString().equals(t.get(colPos).toString())); - //doing String comps here as value objects in Hive in Pig are different so equals() doesn't work } colPos++; } @@ -633,10 +679,10 @@ public class TestHCatLoader { } private static void setupAllTypesTable(Driver driver) throws Exception { String[] primitiveData = new String[primitiveRows.length]; - for(int i = 0; i < primitiveRows.length; i++) { + for (int i = 0; i < primitiveRows.length; i++) { Object[] rowData = primitiveRows[i]; StringBuilder row = new StringBuilder(); - for(Object cell : rowData) { + for (Object cell : rowData) { row.append(row.length() == 0 ? "" : "\t").append(cell == null ? null : cell); } primitiveData[i] = row.toString(); Modified: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1628661&r1=1628660&r2=1628661&view=diff ============================================================================== --- hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java (original) +++ hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java Wed Oct 1 09:44:12 2014 @@ -18,19 +18,25 @@ */ package org.apache.hive.hcatalog.pig; +import com.google.common.collect.ImmutableSet; + import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; - -import junit.framework.Assert; +import java.util.Map; +import java.util.Set; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.StorageFormats; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hadoop.hive.ql.session.SessionState; @@ -46,12 +52,20 @@ import org.apache.pig.impl.logicalLayer. import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; +import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assume.assumeTrue; + +@RunWith(Parameterized.class) public class TestHCatLoaderComplexSchema { //private static MiniCluster cluster = MiniCluster.buildCluster(); @@ -59,13 +73,33 @@ public class TestHCatLoaderComplexSchema //private static Properties props; private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoaderComplexSchema.class); - private void dropTable(String tablename) throws IOException, CommandNeedRetryException { - driver.run("drop table " + tablename); + private static final Map> DISABLED_STORAGE_FORMATS = + new HashMap>() {{ + put(IOConstants.AVRO, new HashSet() {{ + add("testSyntheticComplexSchema"); + add("testTupleInBagInTupleInBag"); + add("testMapWithComplexData"); + }}); + put(IOConstants.PARQUETFILE, new HashSet() {{ + add("testSyntheticComplexSchema"); + add("testTupleInBagInTupleInBag"); + add("testMapWithComplexData"); + }}); + }}; + + private String storageFormat; + + @Parameterized.Parameters + public static Collection generateParameters() { + return StorageFormats.names(); + } + + public TestHCatLoaderComplexSchema(String storageFormat) { + this.storageFormat = storageFormat; } - protected String storageFormat() { - return "RCFILE tblproperties('hcat.isd'='org.apache.hive.hcatalog.rcfile.RCFileInputDriver'," + - "'hcat.osd'='org.apache.hive.hcatalog.rcfile.RCFileOutputDriver')"; + private void dropTable(String tablename) throws IOException, CommandNeedRetryException { + driver.run("drop table " + tablename); } private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException { @@ -74,7 +108,7 @@ public class TestHCatLoaderComplexSchema if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) { createTable = createTable + "partitioned by (" + partitionedBy + ") "; } - createTable = createTable + "stored as " + storageFormat(); + createTable = createTable + "stored as " + storageFormat; LOG.info("Creating table:\n {}", createTable); CommandProcessorResponse result = driver.run(createTable); int retCode = result.getResponseCode(); @@ -89,7 +123,6 @@ public class TestHCatLoaderComplexSchema @BeforeClass public static void setUpBeforeClass() throws Exception { - HiveConf hiveConf = new HiveConf(TestHCatLoaderComplexSchema.class); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); @@ -98,7 +131,6 @@ public class TestHCatLoaderComplexSchema SessionState.start(new CliSessionState(hiveConf)); //props = new Properties(); //props.setProperty("fs.default.name", cluster.getProperties().getProperty("fs.default.name")); - } private static final TupleFactory tf = TupleFactory.getInstance(); @@ -118,6 +150,7 @@ public class TestHCatLoaderComplexSchema */ @Test public void testSyntheticComplexSchema() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); String pigSchema = "a: " + "(" + @@ -186,7 +219,6 @@ public class TestHCatLoaderComplexSchema verifyWriteRead("testSyntheticComplexSchema", pigSchema, tableSchema, data, false); verifyWriteRead("testSyntheticComplexSchema2", pigSchema, tableSchema2, data, true); verifyWriteRead("testSyntheticComplexSchema2", pigSchema, tableSchema2, data, false); - } private void verifyWriteRead(String tablename, String pigSchema, String tableSchema, List data, boolean provideSchemaToStorer) @@ -219,7 +251,7 @@ public class TestHCatLoaderComplexSchema } Schema dumpedXSchema = server.dumpSchema("X"); - Assert.assertEquals( + assertEquals( "expected " + dumpedASchema + " but was " + dumpedXSchema + " (ignoring field names)", "", compareIgnoreFiledNames(dumpedASchema, dumpedXSchema)); @@ -230,14 +262,14 @@ public class TestHCatLoaderComplexSchema } private void compareTuples(Tuple t1, Tuple t2) throws ExecException { - Assert.assertEquals("Tuple Sizes don't match", t1.size(), t2.size()); + assertEquals("Tuple Sizes don't match", t1.size(), t2.size()); for (int i = 0; i < t1.size(); i++) { Object f1 = t1.get(i); Object f2 = t2.get(i); - Assert.assertNotNull("left", f1); - Assert.assertNotNull("right", f2); + assertNotNull("left", f1); + assertNotNull("right", f2); String msg = "right: " + f1 + ", left: " + f2; - Assert.assertEquals(msg, noOrder(f1.toString()), noOrder(f2.toString())); + assertEquals(msg, noOrder(f1.toString()), noOrder(f2.toString())); } } @@ -278,6 +310,7 @@ public class TestHCatLoaderComplexSchema */ @Test public void testTupleInBagInTupleInBag() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); String pigSchema = "a: { b : ( c: { d: (i : long) } ) }"; String tableSchema = "a array< array< bigint > >"; @@ -297,11 +330,11 @@ public class TestHCatLoaderComplexSchema verifyWriteRead("TupleInBagInTupleInBag3", pigSchema, tableSchema2, data, true); verifyWriteRead("TupleInBagInTupleInBag4", pigSchema, tableSchema2, data, false); - } @Test public void testMapWithComplexData() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); String pigSchema = "a: long, b: map[]"; String tableSchema = "a bigint, b map>"; @@ -320,6 +353,5 @@ public class TestHCatLoaderComplexSchema } verifyWriteRead("testMapWithComplexData", pigSchema, tableSchema, data, true); verifyWriteRead("testMapWithComplexData2", pigSchema, tableSchema, data, false); - } } Modified: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java?rev=1628661&r1=1628660&r2=1628661&view=diff ============================================================================== --- hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java (original) +++ hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java Wed Oct 1 09:44:12 2014 @@ -18,18 +18,27 @@ */ package org.apache.hive.hcatalog.pig; +import com.google.common.collect.ImmutableSet; + import java.io.BufferedReader; import java.io.File; import java.io.FileReader; import java.io.IOException; import java.math.BigDecimal; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.Set; import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.StorageFormats; import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse; import org.apache.hive.hcatalog.HcatTestUtils; @@ -47,20 +56,92 @@ import org.apache.pig.impl.util.LogUtils import org.joda.time.DateTime; import org.joda.time.DateTimeZone; -import org.junit.Assert; +import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import static org.junit.Assert.*; +import static org.junit.Assume.assumeTrue; + +@RunWith(Parameterized.class) public class TestHCatStorer extends HCatBaseTest { private static final Logger LOG = LoggerFactory.getLogger(TestHCatStorer.class); private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; + private static final Map> DISABLED_STORAGE_FORMATS = + new HashMap>() {{ + put(IOConstants.AVRO, new HashSet() {{ + add("testBagNStruct"); + add("testDateCharTypes"); + add("testDynamicPartitioningMultiPartColsInDataNoSpec"); + add("testDynamicPartitioningMultiPartColsInDataPartialSpec"); + add("testMultiPartColsInData"); + add("testPartColsInData"); + add("testStoreFuncAllSimpleTypes"); + add("testStoreFuncSimple"); + add("testStoreInPartiitonedTbl"); + add("testStoreMultiTables"); + add("testStoreWithNoCtorArgs"); + add("testStoreWithNoSchema"); + add("testWriteChar"); + add("testWriteDate"); + add("testWriteDate2"); + add("testWriteDate3"); + add("testWriteDecimal"); + add("testWriteDecimalX"); + add("testWriteDecimalXY"); + add("testWriteSmallint"); + add("testWriteTimestamp"); + add("testWriteTinyint"); + add("testWriteVarchar"); + }}); + put(IOConstants.PARQUETFILE, new HashSet() {{ + add("testBagNStruct"); + add("testDateCharTypes"); + add("testDynamicPartitioningMultiPartColsInDataNoSpec"); + add("testDynamicPartitioningMultiPartColsInDataPartialSpec"); + add("testMultiPartColsInData"); + add("testPartColsInData"); + add("testStoreFuncAllSimpleTypes"); + add("testStoreFuncSimple"); + add("testStoreInPartiitonedTbl"); + add("testStoreMultiTables"); + add("testStoreWithNoCtorArgs"); + add("testStoreWithNoSchema"); + add("testWriteChar"); + add("testWriteDate"); + add("testWriteDate2"); + add("testWriteDate3"); + add("testWriteDecimal"); + add("testWriteDecimalX"); + add("testWriteDecimalXY"); + add("testWriteSmallint"); + add("testWriteTimestamp"); + add("testWriteTinyint"); + add("testWriteVarchar"); + }}); + }}; + + private String storageFormat; + + @Parameterized.Parameters + public static Collection generateParameters() { + return StorageFormats.names(); + } + + public TestHCatStorer(String storageFormat) { + this.storageFormat = storageFormat; + } + //Start: tests that check values from Pig that are out of range for target column @Test public void testWriteTinyint() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); pigValueRangeTest("junitTypeTest1", "tinyint", "int", null, Integer.toString(1), Integer.toString(1)); pigValueRangeTestOverflow("junitTypeTest1", "tinyint", "int", null, Integer.toString(300)); pigValueRangeTestOverflow("junitTypeTest2", "tinyint", "int", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Null, @@ -71,6 +152,7 @@ public class TestHCatStorer extends HCat @Test public void testWriteSmallint() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); pigValueRangeTest("junitTypeTest1", "smallint", "int", null, Integer.toString(Short.MIN_VALUE), Integer.toString(Short.MIN_VALUE)); pigValueRangeTestOverflow("junitTypeTest2", "smallint", "int", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Null, @@ -81,6 +163,7 @@ public class TestHCatStorer extends HCat @Test public void testWriteChar() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); pigValueRangeTest("junitTypeTest1", "char(5)", "chararray", null, "xxx", "xxx "); pigValueRangeTestOverflow("junitTypeTest1", "char(5)", "chararray", null, "too_long"); pigValueRangeTestOverflow("junitTypeTest2", "char(5)", "chararray", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Null, @@ -91,6 +174,7 @@ public class TestHCatStorer extends HCat @Test public void testWriteVarchar() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); pigValueRangeTest("junitTypeTest1", "varchar(5)", "chararray", null, "xxx", "xxx"); pigValueRangeTestOverflow("junitTypeTest1", "varchar(5)", "chararray", null, "too_long"); pigValueRangeTestOverflow("junitTypeTest2", "varchar(5)", "chararray", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Null, @@ -101,6 +185,7 @@ public class TestHCatStorer extends HCat @Test public void testWriteDecimalXY() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); pigValueRangeTest("junitTypeTest1", "decimal(5,2)", "bigdecimal", null, BigDecimal.valueOf(1.2).toString(), BigDecimal.valueOf(1.2).toString()); pigValueRangeTestOverflow("junitTypeTest1", "decimal(5,2)", "bigdecimal", null, BigDecimal.valueOf(12345.12).toString()); @@ -112,6 +197,7 @@ public class TestHCatStorer extends HCat @Test public void testWriteDecimalX() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); //interestingly decimal(2) means decimal(2,0) pigValueRangeTest("junitTypeTest1", "decimal(2)", "bigdecimal", null, BigDecimal.valueOf(12).toString(), BigDecimal.valueOf(12).toString()); @@ -123,6 +209,7 @@ public class TestHCatStorer extends HCat @Test public void testWriteDecimal() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); //decimal means decimal(10,0) pigValueRangeTest("junitTypeTest1", "decimal", "bigdecimal", null, BigDecimal.valueOf(1234567890).toString(), BigDecimal.valueOf(1234567890).toString()); @@ -137,8 +224,10 @@ public class TestHCatStorer extends HCat * include time to make sure it's 0 */ private static final String FORMAT_4_DATE = "yyyy-MM-dd HH:mm:ss"; + @Test public void testWriteDate() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); DateTime d = new DateTime(1991,10,11,0,0); pigValueRangeTest("junitTypeTest1", "date", "datetime", null, d.toString(), d.toString(FORMAT_4_DATE), FORMAT_4_DATE); @@ -157,6 +246,7 @@ public class TestHCatStorer extends HCat @Test public void testWriteDate3() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); DateTime d = new DateTime(1991,10,11,23,10,DateTimeZone.forOffsetHours(-11)); FrontendException fe = null; //expect to fail since the time component is not 0 @@ -170,6 +260,7 @@ public class TestHCatStorer extends HCat @Test public void testWriteDate2() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); DateTime d = new DateTime(1991,11,12,0,0, DateTimeZone.forID("US/Eastern")); pigValueRangeTest("junitTypeTest1", "date", "datetime", null, d.toString(), d.toString(FORMAT_4_DATE), FORMAT_4_DATE); @@ -193,6 +284,7 @@ public class TestHCatStorer extends HCat */ @Test public void testWriteTimestamp() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); DateTime d = new DateTime(1991,10,11,14,23,30, 10);//uses default TZ pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(), d.toDateTime(DateTimeZone.getDefault()).toString()); @@ -229,13 +321,6 @@ public class TestHCatStorer extends HCat } /** - * this should be overridden in subclass to test with different file formats - */ - String getStorageFormat() { - return "RCFILE"; - } - - /** * This is used to test how Pig values of various data types which are out of range for Hive target * column are handled. Currently the options are to raise an error or write NULL. * 1. create a data file with 1 column, 1 row @@ -258,7 +343,7 @@ public class TestHCatStorer extends HCat throws Exception { TestHCatLoader.dropTable(tblName, driver); final String field = "f1"; - TestHCatLoader.createTable(tblName, field + " " + hiveType, null, driver, getStorageFormat()); + TestHCatLoader.createTable(tblName, field + " " + hiveType, null, driver, storageFormat); HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, new String[] {inputValue}); LOG.debug("File=" + INPUT_FILE_NAME); dumpFile(INPUT_FILE_NAME); @@ -287,11 +372,11 @@ public class TestHCatStorer extends HCat //do nothing, fall through and verify the data break; case Throw: - Assert.assertTrue("Expected a FrontendException", fe != null); - Assert.assertEquals("Expected a different FrontendException.", fe.getMessage(), "Unable to store alias A"); + assertTrue("Expected a FrontendException", fe != null); + assertEquals("Expected a different FrontendException.", fe.getMessage(), "Unable to store alias A"); return;//this test is done default: - Assert.assertFalse("Unexpected goal: " + goal, 1 == 1); + assertFalse("Unexpected goal: " + goal, 1 == 1); } } logAndRegister(server, "B = load '" + tblName + "' using " + HCatLoader.class.getName() + "();", queryNumber); @@ -310,17 +395,17 @@ public class TestHCatStorer extends HCat Tuple t = itr.next(); if("date".equals(hiveType)) { DateTime dateTime = (DateTime)t.get(0); - Assert.assertTrue(format != null); - Assert.assertEquals("Comparing Pig to Raw data for table " + tblName, expectedValue, dateTime== null ? null : dateTime.toString(format)); + assertTrue(format != null); + assertEquals("Comparing Pig to Raw data for table " + tblName, expectedValue, dateTime== null ? null : dateTime.toString(format)); } else { - Assert.assertEquals("Comparing Pig to Raw data for table " + tblName, expectedValue, t.isNull(0) ? null : t.get(0).toString()); + assertEquals("Comparing Pig to Raw data for table " + tblName, expectedValue, t.isNull(0) ? null : t.get(0).toString()); } //see comment at "Dumping rows via SQL..." for why this doesn't work - //Assert.assertEquals("Comparing Pig to Hive", t.get(0), l.get(0)); + //assertEquals("Comparing Pig to Hive", t.get(0), l.get(0)); numRowsRead++; } - Assert.assertEquals("Expected " + 1 + " rows; got " + numRowsRead + " file=" + INPUT_FILE_NAME + "; table " + + assertEquals("Expected " + 1 + " rows; got " + numRowsRead + " file=" + INPUT_FILE_NAME + "; table " + tblName, 1, numRowsRead); /* Misc notes: Unfortunately Timestamp.toString() adjusts the value for local TZ and 't' is a String @@ -334,10 +419,11 @@ public class TestHCatStorer extends HCat */ @Test public void testDateCharTypes() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); final String tblName = "junit_date_char"; TestHCatLoader.dropTable(tblName, driver); TestHCatLoader.createTable(tblName, - "id int, char5 char(5), varchar10 varchar(10), dec52 decimal(5,2)", null, driver, getStorageFormat()); + "id int, char5 char(5), varchar10 varchar(10), dec52 decimal(5,2)", null, driver, storageFormat); int NUM_ROWS = 5; String[] rows = new String[NUM_ROWS]; for(int i = 0; i < NUM_ROWS; i++) { @@ -376,12 +462,12 @@ public class TestHCatStorer extends HCat rowFromPig.append(t.get(i)).append("\t"); } rowFromPig.setLength(rowFromPig.length() - 1); - Assert.assertEquals("Comparing Pig to Raw data", rows[numRowsRead], rowFromPig.toString()); + assertEquals("Comparing Pig to Raw data", rows[numRowsRead], rowFromPig.toString()); //see comment at "Dumping rows via SQL..." for why this doesn't work (for all types) - //Assert.assertEquals("Comparing Pig to Hive", rowFromPig.toString(), l.get(numRowsRead)); + //assertEquals("Comparing Pig to Hive", rowFromPig.toString(), l.get(numRowsRead)); numRowsRead++; } - Assert.assertEquals("Expected " + NUM_ROWS + " rows; got " + numRowsRead + " file=" + INPUT_FILE_NAME, NUM_ROWS, numRowsRead); + assertEquals("Expected " + NUM_ROWS + " rows; got " + numRowsRead + " file=" + INPUT_FILE_NAME, NUM_ROWS, numRowsRead); } static void dumpFile(String fileName) throws Exception { @@ -397,9 +483,10 @@ public class TestHCatStorer extends HCat @Test public void testPartColsInData() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as " + getStorageFormat(); + String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -420,22 +507,23 @@ public class TestHCatStorer extends HCat while (itr.hasNext()) { Tuple t = itr.next(); - Assert.assertEquals(2, t.size()); - Assert.assertEquals(t.get(0), i); - Assert.assertEquals(t.get(1), "1"); + assertEquals(2, t.size()); + assertEquals(t.get(0), i); + assertEquals(t.get(1), "1"); i++; } - Assert.assertFalse(itr.hasNext()); - Assert.assertEquals(LOOP_SIZE, i); + assertFalse(itr.hasNext()); + assertEquals(LOOP_SIZE, i); } @Test public void testMultiPartColsInData() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table employee"); String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " + - " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + getStorageFormat(); + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { @@ -464,20 +552,21 @@ public class TestHCatStorer extends HCat driver.run("select * from employee"); ArrayList results = new ArrayList(); driver.getResults(results); - Assert.assertEquals(4, results.size()); + assertEquals(4, results.size()); Collections.sort(results); - Assert.assertEquals(inputData[0], results.get(0)); - Assert.assertEquals(inputData[1], results.get(1)); - Assert.assertEquals(inputData[2], results.get(2)); - Assert.assertEquals(inputData[3], results.get(3)); + assertEquals(inputData[0], results.get(0)); + assertEquals(inputData[1], results.get(1)); + assertEquals(inputData[2], results.get(2)); + assertEquals(inputData[3], results.get(3)); driver.run("drop table employee"); } @Test public void testStoreInPartiitonedTbl() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as " + getStorageFormat(); + String createTable = "create table junit_unparted(a int) partitioned by (b string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -498,20 +587,21 @@ public class TestHCatStorer extends HCat while (itr.hasNext()) { Tuple t = itr.next(); - Assert.assertEquals(2, t.size()); - Assert.assertEquals(t.get(0), i); - Assert.assertEquals(t.get(1), "1"); + assertEquals(2, t.size()); + assertEquals(t.get(0), i); + assertEquals(t.get(1), "1"); i++; } - Assert.assertFalse(itr.hasNext()); - Assert.assertEquals(11, i); + assertFalse(itr.hasNext()); + assertEquals(11, i); } @Test public void testNoAlias() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_parted"); - String createTable = "create table junit_parted(a int, b string) partitioned by (ds string) stored as " + getStorageFormat(); + String createTable = "create table junit_parted(a int, b string) partitioned by (ds string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -526,12 +616,12 @@ public class TestHCatStorer extends HCat server.executeBatch(); } catch (PigException fe) { PigException pe = LogUtils.getPigException(fe); - Assert.assertTrue(pe instanceof FrontendException); - Assert.assertEquals(PigHCatUtil.PIG_EXCEPTION_CODE, pe.getErrorCode()); - Assert.assertTrue(pe.getMessage().contains("Column name for a field is not specified. Please provide the full schema as an argument to HCatStorer.")); + assertTrue(pe instanceof FrontendException); + assertEquals(PigHCatUtil.PIG_EXCEPTION_CODE, pe.getErrorCode()); + assertTrue(pe.getMessage().contains("Column name for a field is not specified. Please provide the full schema as an argument to HCatStorer.")); errCaught = true; } - Assert.assertTrue(errCaught); + assertTrue(errCaught); errCaught = false; try { server.setBatchOn(); @@ -541,20 +631,21 @@ public class TestHCatStorer extends HCat server.executeBatch(); } catch (PigException fe) { PigException pe = LogUtils.getPigException(fe); - Assert.assertTrue(pe instanceof FrontendException); - Assert.assertEquals(PigHCatUtil.PIG_EXCEPTION_CODE, pe.getErrorCode()); - Assert.assertTrue(pe.getMessage().contains("Column names should all be in lowercase. Invalid name found: B")); + assertTrue(pe instanceof FrontendException); + assertEquals(PigHCatUtil.PIG_EXCEPTION_CODE, pe.getErrorCode()); + assertTrue(pe.getMessage().contains("Column names should all be in lowercase. Invalid name found: B")); errCaught = true; } driver.run("drop table junit_parted"); - Assert.assertTrue(errCaught); + assertTrue(errCaught); } @Test public void testStoreMultiTables() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); + String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -598,18 +689,19 @@ public class TestHCatStorer extends HCat Iterator itr = res.iterator(); for (int i = 0; i < LOOP_SIZE * LOOP_SIZE; i++) { - Assert.assertEquals(input[i], itr.next()); + assertEquals(input[i], itr.next()); } - Assert.assertFalse(itr.hasNext()); + assertFalse(itr.hasNext()); } @Test public void testStoreWithNoSchema() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); + String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -637,18 +729,19 @@ public class TestHCatStorer extends HCat driver.run("drop table junit_unparted"); Iterator itr = res.iterator(); for (int i = 0; i < LOOP_SIZE * LOOP_SIZE; i++) { - Assert.assertEquals(input[i], itr.next()); + assertEquals(input[i], itr.next()); } - Assert.assertFalse(itr.hasNext()); + assertFalse(itr.hasNext()); } @Test public void testStoreWithNoCtorArgs() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); + String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -676,18 +769,19 @@ public class TestHCatStorer extends HCat driver.run("drop table junit_unparted"); Iterator itr = res.iterator(); for (int i = 0; i < LOOP_SIZE * LOOP_SIZE; i++) { - Assert.assertEquals(input[i], itr.next()); + assertEquals(input[i], itr.next()); } - Assert.assertFalse(itr.hasNext()); + assertFalse(itr.hasNext()); } @Test public void testEmptyStore() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); + String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -715,15 +809,16 @@ public class TestHCatStorer extends HCat driver.getResults(res); driver.run("drop table junit_unparted"); Iterator itr = res.iterator(); - Assert.assertFalse(itr.hasNext()); + assertFalse(itr.hasNext()); } @Test public void testBagNStruct() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); String createTable = "create table junit_unparted(b string,a struct, arr_of_struct array, " + - "arr_of_struct2 array>, arr_of_struct3 array>) stored as " + getStorageFormat(); + "arr_of_struct2 array>, arr_of_struct3 array>) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -746,17 +841,18 @@ public class TestHCatStorer extends HCat driver.getResults(res); driver.run("drop table junit_unparted"); Iterator itr = res.iterator(); - Assert.assertEquals("zookeeper\t{\"a1\":2}\t[\"pig\"]\t[{\"s1\":\"pnuts\",\"s2\":\"hdfs\"}]\t[{\"s3\":\"hadoop\"},{\"s3\":\"hcat\"}]", itr.next()); - Assert.assertEquals("chubby\t{\"a1\":2}\t[\"sawzall\"]\t[{\"s1\":\"bigtable\",\"s2\":\"gfs\"}]\t[{\"s3\":\"mapreduce\"},{\"s3\":\"hcat\"}]", itr.next()); - Assert.assertFalse(itr.hasNext()); + assertEquals("zookeeper\t{\"a1\":2}\t[\"pig\"]\t[{\"s1\":\"pnuts\",\"s2\":\"hdfs\"}]\t[{\"s3\":\"hadoop\"},{\"s3\":\"hcat\"}]", itr.next()); + assertEquals("chubby\t{\"a1\":2}\t[\"sawzall\"]\t[{\"s1\":\"bigtable\",\"s2\":\"gfs\"}]\t[{\"s3\":\"mapreduce\"},{\"s3\":\"hcat\"}]", itr.next()); + assertFalse(itr.hasNext()); } @Test public void testStoreFuncAllSimpleTypes() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as " + getStorageFormat(); + String createTable = "create table junit_unparted(a int, b float, c double, d bigint, e string, h boolean, f binary, g binary) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -783,10 +879,10 @@ public class TestHCatStorer extends HCat Iterator itr = res.iterator(); String next = itr.next(); - Assert.assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL", next ); - Assert.assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\ttrue\tbinary-data\tNULL", itr.next()); - Assert.assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tfalse\tbinary-data\tNULL", itr.next()); - Assert.assertFalse(itr.hasNext()); + assertEquals("0\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL\tNULL", next ); + assertEquals("NULL\t4.2\t2.2\t4\tlets hcat\ttrue\tbinary-data\tNULL", itr.next()); + assertEquals("3\t6.2999997\t3.3000000000000003\t6\tlets hcat\tfalse\tbinary-data\tNULL", itr.next()); + assertFalse(itr.hasNext()); server.registerQuery("B = load 'junit_unparted' using " + HCatLoader.class.getName() + ";"); Iterator iter = server.openIterator("B"); @@ -797,21 +893,22 @@ public class TestHCatStorer extends HCat if (t.get(6) == null) { num5nulls++; } else { - Assert.assertTrue(t.get(6) instanceof DataByteArray); + assertTrue(t.get(6) instanceof DataByteArray); } - Assert.assertNull(t.get(7)); + assertNull(t.get(7)); count++; } - Assert.assertEquals(3, count); - Assert.assertEquals(1, num5nulls); + assertEquals(3, count); + assertEquals(1, num5nulls); driver.run("drop table junit_unparted"); } @Test public void testStoreFuncSimple() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table junit_unparted"); - String createTable = "create table junit_unparted(a int, b string) stored as " + getStorageFormat(); + String createTable = "create table junit_unparted(a int, b string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -841,19 +938,20 @@ public class TestHCatStorer extends HCat for (int i = 1; i <= LOOP_SIZE; i++) { String si = i + ""; for (int j = 1; j <= LOOP_SIZE; j++) { - Assert.assertEquals(si + "\t" + j, itr.next()); + assertEquals(si + "\t" + j, itr.next()); } } - Assert.assertFalse(itr.hasNext()); + assertFalse(itr.hasNext()); } @Test public void testDynamicPartitioningMultiPartColsInDataPartialSpec() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table if exists employee"); String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " + - " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + getStorageFormat(); + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { @@ -876,21 +974,22 @@ public class TestHCatStorer extends HCat driver.run("select * from employee"); ArrayList results = new ArrayList(); driver.getResults(results); - Assert.assertEquals(4, results.size()); + assertEquals(4, results.size()); Collections.sort(results); - Assert.assertEquals(inputData[0], results.get(0)); - Assert.assertEquals(inputData[1], results.get(1)); - Assert.assertEquals(inputData[2], results.get(2)); - Assert.assertEquals(inputData[3], results.get(3)); + assertEquals(inputData[0], results.get(0)); + assertEquals(inputData[1], results.get(1)); + assertEquals(inputData[2], results.get(2)); + assertEquals(inputData[3], results.get(3)); driver.run("drop table employee"); } @Test public void testDynamicPartitioningMultiPartColsInDataNoSpec() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table if exists employee"); String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " + - " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + getStorageFormat(); + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { @@ -913,21 +1012,22 @@ public class TestHCatStorer extends HCat driver.run("select * from employee"); ArrayList results = new ArrayList(); driver.getResults(results); - Assert.assertEquals(4, results.size()); + assertEquals(4, results.size()); Collections.sort(results); - Assert.assertEquals(inputData[0], results.get(0)); - Assert.assertEquals(inputData[1], results.get(1)); - Assert.assertEquals(inputData[2], results.get(2)); - Assert.assertEquals(inputData[3], results.get(3)); + assertEquals(inputData[0], results.get(0)); + assertEquals(inputData[1], results.get(1)); + assertEquals(inputData[2], results.get(2)); + assertEquals(inputData[3], results.get(3)); driver.run("drop table employee"); } @Test public void testDynamicPartitioningMultiPartColsNoDataInDataNoSpec() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table if exists employee"); String createTable = "CREATE TABLE employee (emp_id INT, emp_name STRING, emp_start_date STRING , emp_gender STRING ) " + - " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + getStorageFormat(); + " PARTITIONED BY (emp_country STRING , emp_state STRING ) STORED AS " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { @@ -947,15 +1047,16 @@ public class TestHCatStorer extends HCat driver.run("select * from employee"); ArrayList results = new ArrayList(); driver.getResults(results); - Assert.assertEquals(0, results.size()); + assertEquals(0, results.size()); driver.run("drop table employee"); } + @Test - public void testPartitionPublish() - throws IOException, CommandNeedRetryException { + public void testPartitionPublish() throws IOException, CommandNeedRetryException { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); driver.run("drop table ptn_fail"); - String createTable = "create table ptn_fail(a int, c string) partitioned by (b string) stored as " + getStorageFormat(); + String createTable = "create table ptn_fail(a int, c string) partitioned by (b string) stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table."); @@ -987,11 +1088,11 @@ public class TestHCatStorer extends HCat ArrayList res = new ArrayList(); driver.getResults(res); - Assert.assertEquals(0, res.size()); + assertEquals(0, res.size()); // Make sure the partitions directory is not in hdfs. - Assert.assertTrue((new File(TEST_WAREHOUSE_DIR + "/ptn_fail")).exists()); - Assert.assertFalse((new File(TEST_WAREHOUSE_DIR + "/ptn_fail/b=math")) + assertTrue((new File(TEST_WAREHOUSE_DIR + "/ptn_fail")).exists()); + assertFalse((new File(TEST_WAREHOUSE_DIR + "/ptn_fail/b=math")) .exists()); } Modified: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java?rev=1628661&r1=1628660&r2=1628661&view=diff ============================================================================== --- hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java (original) +++ hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java Wed Oct 1 09:44:12 2014 @@ -18,18 +18,25 @@ */ package org.apache.hive.hcatalog.pig; +import com.google.common.collect.ImmutableSet; + import java.io.File; import java.io.FileWriter; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hive.cli.CliSessionState; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CommandNeedRetryException; import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.io.IOConstants; +import org.apache.hadoop.hive.ql.io.StorageFormats; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hive.hcatalog.common.HCatUtil; @@ -41,13 +48,17 @@ import org.apache.pig.PigServer; import org.junit.After; import org.junit.Before; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; +@RunWith(Parameterized.class) public class TestHCatStorerMulti { public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName( - System.getProperty("user.dir") + "/build/test/data/" + - TestHCatStorerMulti.class.getCanonicalName() + "-" + System.currentTimeMillis()); + System.getProperty("user.dir") + "/build/test/data/" + + TestHCatStorerMulti.class.getCanonicalName() + "-" + System.currentTimeMillis()); private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse"; private static final String INPUT_FILE_NAME = TEST_DATA_DIR + "/input.data"; @@ -57,9 +68,29 @@ public class TestHCatStorerMulti { private static Map> basicInputData; - protected String storageFormat() { - return "RCFILE tblproperties('hcat.isd'='org.apache.hive.hcatalog.rcfile.RCFileInputDriver'," + - "'hcat.osd'='org.apache.hive.hcatalog.rcfile.RCFileOutputDriver')"; + private static final Map> DISABLED_STORAGE_FORMATS = + new HashMap>() {{ + put(IOConstants.AVRO, new HashSet() {{ + add("testStoreBasicTable"); + add("testStorePartitionedTable"); + add("testStoreTableMulti"); + }}); + put(IOConstants.PARQUETFILE, new HashSet() {{ + add("testStoreBasicTable"); + add("testStorePartitionedTable"); + add("testStoreTableMulti"); + }}); + }}; + + private String storageFormat; + + @Parameterized.Parameters + public static Collection generateParameters() { + return StorageFormats.names(); + } + + public TestHCatStorerMulti(String storageFormat) { + this.storageFormat = storageFormat; } private void dropTable(String tablename) throws IOException, CommandNeedRetryException { @@ -72,7 +103,7 @@ public class TestHCatStorerMulti { if ((partitionedBy != null) && (!partitionedBy.trim().isEmpty())) { createTable = createTable + "partitioned by (" + partitionedBy + ") "; } - createTable = createTable + "stored as " + storageFormat(); + createTable = createTable + "stored as " + storageFormat; int retCode = driver.run(createTable).getResponseCode(); if (retCode != 0) { throw new IOException("Failed to create table. [" + createTable + "], return code from hive driver : [" + retCode + "]"); @@ -85,6 +116,8 @@ public class TestHCatStorerMulti { @Before public void setUp() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); + if (driver == null) { HiveConf hiveConf = new HiveConf(this.getClass()); hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); @@ -105,6 +138,7 @@ public class TestHCatStorerMulti { @Test public void testStoreBasicTable() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); createTable(BASIC_TABLE, "a int, b string"); populateBasicFile(); @@ -124,6 +158,7 @@ public class TestHCatStorerMulti { @Test public void testStorePartitionedTable() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); createTable(PARTITIONED_TABLE, "a int, b string", "bkt string"); populateBasicFile(); @@ -147,6 +182,7 @@ public class TestHCatStorerMulti { @Test public void testStoreTableMulti() throws Exception { + assumeTrue(!TestUtil.shouldSkip(storageFormat, DISABLED_STORAGE_FORMATS)); createTable(BASIC_TABLE, "a int, b string"); createTable(PARTITIONED_TABLE, "a int, b string", "bkt string"); Added: hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestUtil.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestUtil.java?rev=1628661&view=auto ============================================================================== --- hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestUtil.java (added) +++ hive/branches/branch-0.14/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestUtil.java Wed Oct 1 09:44:12 2014 @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hive.hcatalog.pig; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; + +/** + * Test utilities for selectively disabling specific test methods for given storage formats. + */ +public class TestUtil { + private static final Map> SAMPLE_DISABLED_TESTS_MAP = + new HashMap>() {{ + put("test", new HashSet() {{ + add("testShouldSkip"); + }}); + }}; + + /** + * Determine whether the caller test method is in a set of disabled test methods for a given + * storage format. + * + * @param storageFormat The name of the storage format used in a STORED AS clause. + * @param disabledTestsMap Map of storage format name to set of test method names that indicate + * which test methods should not run against the given storage format. + * @return True if the caller test method should be skipped for the given storage format. + */ + public static boolean shouldSkip(String storageFormat, Map> disabledTestsMap) { + final StackTraceElement[] elements = Thread.currentThread().getStackTrace(); + // The "bottom" of the call stack is at the front of the array. The elements are as follows: + // [0] getStackTrace() + // [1] shouldSkip() + // [2] caller test method + String methodName = elements[2].getMethodName(); + if (!disabledTestsMap.containsKey(storageFormat)) { + return false; + } + + Set disabledMethods = disabledTestsMap.get(storageFormat); + return disabledMethods.contains(methodName); + } + + @Test + public void testShouldSkip() { + assertTrue(TestUtil.shouldSkip("test", SAMPLE_DISABLED_TESTS_MAP)); + } + + @Test + public void testShouldNotSkip() { + assertFalse(TestUtil.shouldSkip("test", SAMPLE_DISABLED_TESTS_MAP)); + assertFalse(TestUtil.shouldSkip("foo", SAMPLE_DISABLED_TESTS_MAP)); + } +} Modified: hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java?rev=1628661&r1=1628660&r2=1628661&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java (original) +++ hive/branches/branch-0.14/ql/src/test/org/apache/hadoop/hive/ql/io/StorageFormats.java Wed Oct 1 09:44:12 2014 @@ -90,7 +90,9 @@ public class StorageFormats { * includes both native Hive storage formats as well as those enumerated in the * ADDITIONAL_STORAGE_FORMATS table. * - * @return List of storage format as paramters. + * @return List of storage format as a Collection of Object arrays, each containing (in order): + * Storage format name, SerDe class name, InputFormat class name, OutputFormat class name. + * This list is used as the parameters to JUnit parameterized tests. */ public static Collection asParameters() { List parameters = new ArrayList(); @@ -130,5 +132,21 @@ public class StorageFormats { return parameters; } + + /** + * Returns a list of the names of storage formats. + * + * @return List of names of storage formats. + */ + public static Collection names() { + List names = new ArrayList(); + for (StorageFormatDescriptor descriptor : ServiceLoader.load(StorageFormatDescriptor.class)) { + String[] formatNames = new String[descriptor.getNames().size()]; + formatNames = descriptor.getNames().toArray(formatNames); + String[] params = { formatNames[0] }; + names.add(params); + } + return names; + } }