hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From khorg...@apache.org
Subject svn commit: r1581531 - in /hive/trunk/hcatalog/hcatalog-pig-adapter/src: main/java/org/apache/hive/hcatalog/pig/ test/java/org/apache/hive/hcatalog/pig/
Date Tue, 25 Mar 2014 21:25:17 GMT
Author: khorgath
Date: Tue Mar 25 21:25:17 2014
New Revision: 1581531

URL: http://svn.apache.org/r1581531
Log:
HIVE-6724 : HCatStorer throws ClassCastException while storing tinyint/smallint data (Eugene
Koifman via Sushanth Sowmyan)

Modified:
    hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java
    hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
    hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
    hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
    hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java

Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java
(original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/HCatStorer.java
Tue Mar 25 21:25:17 2014
@@ -149,9 +149,6 @@ public class HCatStorer extends HCatBase
    */
   @Override
   public void setStoreLocation(String location, Job job) throws IOException {
-    HCatContext.INSTANCE.setConf(job.getConfiguration()).getConf().get()
-      .setBoolean(HCatConstants.HCAT_DATA_TINY_SMALL_INT_PROMOTION, false);
-
     Configuration config = job.getConfiguration();
     config.set(INNER_SIGNATURE, INNER_SIGNATURE_PREFIX + "_" + sign);
     Properties udfProps = UDFContext.getUDFContext().getUDFProperties(

Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
(original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/main/java/org/apache/hive/hcatalog/pig/PigHCatUtil.java
Tue Mar 25 21:25:17 2014
@@ -381,6 +381,10 @@ class PigHCatUtil {
    * @return object in Pig value system 
    */
   public static Object extractPigObject(Object o, HCatFieldSchema hfs) throws Exception {
+    /*Note that HCatRecordSerDe.serializePrimitiveField() will be called before this, thus
some
+    * type promotion/conversion may occur: e.g. Short to Integer.  We should refactor this
so
+    * that it's hapenning in one place per module/product that we are integrating with.
+    * All Pig conversion should be done here, etc.*/
     if(o == null) {
       return null;
     }

Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
(original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
Tue Mar 25 21:25:17 2014
@@ -94,7 +94,7 @@ public class TestHCatLoader {
     dropTable(tablename, driver);
   }
   static void dropTable(String tablename, Driver driver) throws IOException, CommandNeedRetryException
{
-    driver.run("drop table " + tablename);
+    driver.run("drop table if exists " + tablename);
   }
 
   private void createTable(String tablename, String schema, String partitionedBy) throws
IOException, CommandNeedRetryException {

Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
(original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderStorer.java
Tue Mar 25 21:25:17 2014
@@ -30,8 +30,11 @@ import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.junit.Assert;
 import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.File;
+import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
@@ -40,8 +43,62 @@ import java.util.List;
  * please consider @{link TestHCatLoader} or @{link TestHCatStorer}.
  */
 public class TestHCatLoaderStorer extends HCatBaseTest {
+  private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoaderStorer.class);
 
   /**
+   * Test round trip of smallint/tinyint: Hive->Pig->Hive.  This is a more general
use case in HCatalog:
+   * 'read some data from Hive, process it in Pig, write result back to a Hive table'
+   */
+  @Test
+  public void testReadWrite() throws Exception {
+    final String tblName = "small_ints_table";
+    final String tblName2 = "pig_hcatalog_1";
+    File dataDir = new File(TEST_DATA_DIR + File.separator + "testReadWrite");
+    FileUtil.fullyDelete(dataDir); // Might not exist
+    Assert.assertTrue(dataDir.mkdir());
+    final String INPUT_FILE_NAME = dataDir + "/inputtrw.data";
+
+    TestHCatLoader.dropTable(tblName, driver);
+    HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, new String[]{"40\t1"});
+
+    TestHCatLoader.executeStatementOnDriver("create external table " + tblName +
+      " (my_small_int smallint, my_tiny_int tinyint)" +
+      " row format delimited fields terminated by '\t' stored as textfile location '" +
+      dataDir + "'", driver);
+    TestHCatLoader.dropTable(tblName2, driver);
+    TestHCatLoader.createTable(tblName2, "my_small_int smallint, my_tiny_int tinyint", null,
driver,
+      "textfile");
+
+    LOG.debug("File=" + INPUT_FILE_NAME);
+    TestHCatStorer.dumpFile(INPUT_FILE_NAME);
+    PigServer server = createPigServer(true);
+    try {
+      int queryNumber = 1;
+      logAndRegister(server,
+        "A = load '" + tblName +
+          "' using org.apache.hive.hcatalog.pig.HCatLoader() as (my_small_int:int, my_tiny_int:int);",
+        queryNumber++);
+      logAndRegister(server,
+        "b = foreach A generate my_small_int + my_tiny_int as my_small_int, my_tiny_int;",
+        queryNumber++);
+      logAndRegister(server, "store b into '" + tblName2 +
+        "' using org.apache.hive.hcatalog.pig.HCatStorer();", queryNumber);
+      //perform simple checksum here; make sure nothing got turned to NULL
+      TestHCatLoader.executeStatementOnDriver("select my_small_int from " + tblName2, driver);
+      ArrayList l = new ArrayList();
+      driver.getResults(l);
+      for(Object t : l) {
+        LOG.debug("t=" + t);
+      }
+      Assert.assertEquals("Expected '1' rows; got '" + l.size() + "'", 1, l.size());
+      int result = Integer.parseInt((String)l.get(0));
+      Assert.assertEquals("Expected value '41'; got '" + result + "'", 41, result);
+    }
+    finally {
+      server.shutdown();
+    }
+  }
+  /**
    * Ensure Pig can read/write tinyint/smallint columns.
    */
   @Test
@@ -137,5 +194,5 @@ public class TestHCatLoaderStorer extend
       "store data into 'test_tbl' using org.apache.hive.hcatalog.pig.HCatStorer('','','-onOutOfRangeValue
Throw');");
     List<ExecJob> jobs = server.executeBatch();
     Assert.assertEquals(expectedStatus, jobs.get(0).getStatus());
-  }
+    }
 }

Modified: hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java?rev=1581531&r1=1581530&r2=1581531&view=diff
==============================================================================
--- hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
(original)
+++ hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
Tue Mar 25 21:25:17 2014
@@ -358,7 +358,7 @@ public class TestHCatStorer extends HCat
     }
     Assert.assertEquals("Expected " + NUM_ROWS + " rows; got " + numRowsRead + " file=" +
INPUT_FILE_NAME, NUM_ROWS, numRowsRead);
   }
-  private static void dumpFile(String fileName) throws Exception {
+  static void dumpFile(String fileName) throws Exception {
     File file = new File(fileName);
     BufferedReader reader = new BufferedReader(new FileReader(file));
     String line = null;



Mime
View raw message