hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From gunt...@apache.org
Subject svn commit: r1626122 - in /hive/branches/cbo: ./ data/files/ hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/ hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/ itests/hive-unit/src/test/java/org/apache...
Date Fri, 19 Sep 2014 04:43:00 GMT
Author: gunther
Date: Fri Sep 19 04:42:59 2014
New Revision: 1626122

URL: http://svn.apache.org/r1626122
Log:
Merge latest trunk into cbo branch. (Gunther Hagleitner)

Added:
    hive/branches/cbo/data/files/data_with_escape.txt
      - copied unchanged from r1626121, hive/trunk/data/files/data_with_escape.txt
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java
      - copied unchanged from r1626121, hive/trunk/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorerMulti.java
    hive/branches/cbo/ql/src/test/queries/clientnegative/update_no_such_table.q
      - copied unchanged from r1626121, hive/trunk/ql/src/test/queries/clientnegative/update_no_such_table.q
    hive/branches/cbo/ql/src/test/queries/clientpositive/escape3.q
      - copied unchanged from r1626121, hive/trunk/ql/src/test/queries/clientpositive/escape3.q
    hive/branches/cbo/ql/src/test/results/clientnegative/update_no_such_table.q.out
      - copied unchanged from r1626121, hive/trunk/ql/src/test/results/clientnegative/update_no_such_table.q.out
    hive/branches/cbo/ql/src/test/results/clientpositive/escape3.q.out
      - copied unchanged from r1626121, hive/trunk/ql/src/test/results/clientpositive/escape3.q.out
Removed:
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatPigStorer.java
Modified:
    hive/branches/cbo/   (props changed)
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java
    hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java
    hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
    hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
    hive/branches/cbo/ql/src/test/queries/clientpositive/drop_index.q
    hive/branches/cbo/ql/src/test/results/clientpositive/drop_index.q.out
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java
    hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java

Propchange: hive/branches/cbo/
------------------------------------------------------------------------------
  Merged /hive/trunk:r1626047-1626121

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/MockLoader.java Fri Sep 19 04:42:59 2014
@@ -36,6 +36,7 @@ import org.apache.hadoop.mapreduce.Job;
 import org.apache.hadoop.mapreduce.JobContext;
 import org.apache.hadoop.mapreduce.RecordReader;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
+
 import org.apache.pig.LoadFunc;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit;
 import org.apache.pig.data.Tuple;

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestE2EScenarios.java Fri Sep 19 04:42:59 2014
@@ -23,9 +23,8 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Iterator;
 
-import junit.framework.TestCase;
-
 import org.apache.commons.io.FileUtils;
+
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hive.cli.CliSessionState;
@@ -42,6 +41,7 @@ import org.apache.hadoop.mapreduce.Recor
 import org.apache.hadoop.mapreduce.RecordWriter;
 import org.apache.hadoop.mapreduce.TaskAttemptContext;
 import org.apache.hadoop.mapreduce.TaskAttemptID;
+
 import org.apache.hive.hcatalog.HcatTestUtils;
 import org.apache.hive.hcatalog.common.HCatConstants;
 import org.apache.hive.hcatalog.common.HCatContext;
@@ -51,12 +51,16 @@ import org.apache.hive.hcatalog.mapreduc
 import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat;
 import org.apache.hive.hcatalog.mapreduce.OutputJobInfo;
 import org.apache.hive.hcatalog.mapreduce.HCatMapRedUtil;
+
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.data.Tuple;
 
-public class TestE2EScenarios extends TestCase {
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 
+public class TestE2EScenarios {
   private static final String TEST_DATA_DIR = System.getProperty("java.io.tmpdir") + File.separator
       + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis();
   private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
@@ -69,9 +73,8 @@ public class TestE2EScenarios extends Te
     return "orc";
   }
 
-  @Override
-  protected void setUp() throws Exception {
-
+  @Before
+  public void setUp() throws Exception {
     File f = new File(TEST_WAREHOUSE_DIR);
     if (f.exists()) {
       FileUtil.fullyDelete(f);
@@ -90,8 +93,8 @@ public class TestE2EScenarios extends Te
 
   }
 
-  @Override
-  protected void tearDown() throws Exception {
+  @After
+  public void tearDown() throws Exception {
     try {
       dropTable("inpy");
       dropTable("rc5318");
@@ -146,16 +149,13 @@ public class TestE2EScenarios extends Te
     System.err.println("===");
   }
 
-
   private void copyTable(String in, String out) throws IOException, InterruptedException {
     Job ijob = new Job();
     Job ojob = new Job();
     HCatInputFormat inpy = new HCatInputFormat();
     inpy.setInput(ijob , null, in);
     HCatOutputFormat oupy = new HCatOutputFormat();
-    oupy.setOutput(ojob,
-      OutputJobInfo.create(null, out, new HashMap<String,String>()
-      ));
+    oupy.setOutput(ojob, OutputJobInfo.create(null, out, new HashMap<String,String>()));
 
     // Test HCatContext
 
@@ -207,6 +207,7 @@ public class TestE2EScenarios extends Te
   }
 
 
+  @Test
   public void testReadOrcAndRCFromPig() throws Exception {
     String tableSchema = "ti tinyint, si smallint,i int, bi bigint, f float, d double, b boolean";
 
@@ -224,15 +225,14 @@ public class TestE2EScenarios extends Te
     driverRun("LOAD DATA LOCAL INPATH '"+TEXTFILE_LOCN+"' OVERWRITE INTO TABLE inpy");
 
     // write it out from hive to an rcfile table, and to an orc table
-//        driverRun("insert overwrite table rc5318 select * from inpy");
+    //driverRun("insert overwrite table rc5318 select * from inpy");
     copyTable("inpy","rc5318");
-//        driverRun("insert overwrite table orc5318 select * from inpy");
+    //driverRun("insert overwrite table orc5318 select * from inpy");
     copyTable("inpy","orc5318");
 
     pigDump("inpy");
     pigDump("rc5318");
     pigDump("orc5318");
-
   }
 
 }

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoader.java Fri Sep 19 04:42:59 2014
@@ -34,6 +34,7 @@ import java.util.Map;
 import java.util.Properties;
 
 import org.apache.commons.io.FileUtils;
+
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.fs.Path;
@@ -45,11 +46,13 @@ import org.apache.hadoop.hive.ql.process
 import org.apache.hadoop.hive.ql.session.SessionState;
 import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
 import org.apache.hadoop.mapreduce.Job;
+
 import org.apache.hive.hcatalog.HcatTestUtils;
 import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.common.HCatConstants;
 import org.apache.hive.hcatalog.data.Pair;
 import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
+
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.ResourceStatistics;
@@ -60,10 +63,13 @@ import org.apache.pig.impl.logicalLayer.
 import org.apache.pig.PigRunner;
 import org.apache.pig.tools.pigstats.OutputStats;
 import org.apache.pig.tools.pigstats.PigStats;
+
 import org.joda.time.DateTime;
+
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -72,7 +78,7 @@ import static org.junit.Assert.*;
 public class TestHCatLoader {
   private static final Logger LOG = LoggerFactory.getLogger(TestHCatLoader.class);
   private static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(System.getProperty("java.io.tmpdir") +
-          File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis());
+      File.separator + TestHCatLoader.class.getCanonicalName() + "-" + System.currentTimeMillis());
   private static final String TEST_WAREHOUSE_DIR = TEST_DATA_DIR + "/warehouse";
   private static final String BASIC_FILE_NAME = TEST_DATA_DIR + "/basic.input.data";
   private static final String COMPLEX_FILE_NAME = TEST_DATA_DIR + "/complex.input.data";
@@ -93,6 +99,7 @@ public class TestHCatLoader {
   private void dropTable(String tablename) throws IOException, CommandNeedRetryException {
     dropTable(tablename, driver);
   }
+
   static void dropTable(String tablename, Driver driver) throws IOException, CommandNeedRetryException {
     driver.run("drop table if exists " + tablename);
   }
@@ -100,7 +107,8 @@ public class TestHCatLoader {
   private void createTable(String tablename, String schema, String partitionedBy) throws IOException, CommandNeedRetryException {
     createTable(tablename, schema, partitionedBy, driver, storageFormat());
   }
-  static void createTable(String tablename, String schema, String partitionedBy, Driver driver, String storageFormat) 
+
+  static void createTable(String tablename, String schema, String partitionedBy, Driver driver, String storageFormat)
       throws IOException, CommandNeedRetryException {
     String createTable;
     createTable = "create table " + tablename + "(" + schema + ") ";
@@ -114,6 +122,7 @@ public class TestHCatLoader {
   private void createTable(String tablename, String schema) throws IOException, CommandNeedRetryException {
     createTable(tablename, schema, null);
   }
+
   /**
    * Execute Hive CLI statement
    * @param cmd arbitrary statement to execute
@@ -125,20 +134,20 @@ public class TestHCatLoader {
       throw new IOException("Failed to execute \"" + cmd + "\". Driver returned " + cpr.getResponseCode() + " Error: " + cpr.getErrorMessage());
     }
   }
+
   private static void checkProjection(FieldSchema fs, String expectedName, byte expectedPigType) {
     assertEquals(fs.alias, expectedName);
     assertEquals("Expected " + DataType.findTypeName(expectedPigType) + "; got " +
       DataType.findTypeName(fs.type), expectedPigType, fs.type);
   }
-  
+
   @Before
   public void setup() throws Exception {
-
     File f = new File(TEST_WAREHOUSE_DIR);
     if (f.exists()) {
       FileUtil.fullyDelete(f);
     }
-    if(!(new File(TEST_WAREHOUSE_DIR).mkdirs())) {
+    if (!(new File(TEST_WAREHOUSE_DIR).mkdirs())) {
       throw new RuntimeException("Could not create " + TEST_WAREHOUSE_DIR);
     }
 
@@ -192,7 +201,7 @@ public class TestHCatLoader {
     server.registerQuery("B = foreach A generate a,b;", ++i);
     server.registerQuery("B2 = filter B by a < 2;", ++i);
     server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');", ++i);
-    
+
     server.registerQuery("C = foreach A generate a,b;", ++i);
     server.registerQuery("C2 = filter C by a >= 2;", ++i);
     server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');", ++i);
@@ -470,7 +479,7 @@ public class TestHCatLoader {
       {
         fs.delete(new Path(PIGOUTPUT_DIR), true);
       }
-    }finally {
+    } finally {
       new File(PIG_FILE).delete();
     }
   }
@@ -534,7 +543,7 @@ public class TestHCatLoader {
   }
 
   /**
-   * basic tests that cover each scalar type 
+   * basic tests that cover each scalar type
    * https://issues.apache.org/jira/browse/HIVE-5814
    */
   private static final class AllTypesTable {

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatLoaderComplexSchema.java Fri Sep 19 04:42:59 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.hive.ql.Command
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
 import org.apache.hadoop.hive.ql.session.SessionState;
+
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 import org.apache.pig.backend.executionengine.ExecException;
@@ -44,8 +45,10 @@ import org.apache.pig.data.TupleFactory;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
+
 import org.junit.BeforeClass;
 import org.junit.Test;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -225,7 +228,7 @@ public class TestHCatLoaderComplexSchema
       dropTable(tablename);
     }
   }
-  
+
   private void compareTuples(Tuple t1, Tuple t2) throws ExecException {
     Assert.assertEquals("Tuple Sizes don't match", t1.size(), t2.size());
     for (int i = 0; i < t1.size(); i++) {
@@ -237,7 +240,7 @@ public class TestHCatLoaderComplexSchema
       Assert.assertEquals(msg, noOrder(f1.toString()), noOrder(f2.toString()));
     }
   }
-  
+
   private String noOrder(String s) {
     char[] chars = s.toCharArray();
     Arrays.sort(chars);

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorer.java Fri Sep 19 04:42:59 2014
@@ -31,8 +31,10 @@ import java.util.Properties;
 
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.processors.CommandProcessorResponse;
+
 import org.apache.hive.hcatalog.HcatTestUtils;
 import org.apache.hive.hcatalog.mapreduce.HCatBaseTest;
+
 import org.apache.pig.EvalFunc;
 import org.apache.pig.ExecType;
 import org.apache.pig.PigException;
@@ -41,10 +43,13 @@ import org.apache.pig.data.DataByteArray
 import org.apache.pig.data.Tuple;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.util.LogUtils;
+
 import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+
 import org.junit.Assert;
 import org.junit.Test;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -63,6 +68,7 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest3", "tinyint", "int", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       Integer.toString(300));
   }
+
   @Test
   public void testWriteSmallint() throws Exception {
     pigValueRangeTest("junitTypeTest1", "smallint", "int", null, Integer.toString(Short.MIN_VALUE),
@@ -72,6 +78,7 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest3", "smallint", "int", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       Integer.toString(Short.MAX_VALUE + 1));
   }
+
   @Test
   public void testWriteChar() throws Exception {
     pigValueRangeTest("junitTypeTest1", "char(5)", "chararray", null, "xxx", "xxx  ");
@@ -81,6 +88,7 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest3", "char(5)", "chararray", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       "too_long2");
   }
+
   @Test
   public void testWriteVarchar() throws Exception {
     pigValueRangeTest("junitTypeTest1", "varchar(5)", "chararray", null, "xxx", "xxx");
@@ -90,6 +98,7 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest3", "varchar(5)", "chararray", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       "too_long2");
   }
+
   @Test
   public void testWriteDecimalXY() throws Exception {
     pigValueRangeTest("junitTypeTest1", "decimal(5,2)", "bigdecimal", null, BigDecimal.valueOf(1.2).toString(),
@@ -100,6 +109,7 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest3", "decimal(5,2)", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       BigDecimal.valueOf(500.123).toString());
   }
+
   @Test
   public void testWriteDecimalX() throws Exception {
     //interestingly decimal(2) means decimal(2,0)
@@ -110,6 +120,7 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest3", "decimal(2)", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       BigDecimal.valueOf(50.123).toString());
   }
+
   @Test
   public void testWriteDecimal() throws Exception {
     //decimal means decimal(10,0)
@@ -120,9 +131,10 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest3", "decimal", "bigdecimal", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       BigDecimal.valueOf(12345678900L).toString());
   }
+
   /**
    * because we want to ignore TZ which is included in toString()
-   * include time to make sure it's 0 
+   * include time to make sure it's 0
    */
   private static final String FORMAT_4_DATE = "yyyy-MM-dd HH:mm:ss";
   @Test
@@ -142,6 +154,7 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest6", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       d.plusMinutes(1).toString(), FORMAT_4_DATE);//date out of range due to time!=0
   }
+
   @Test
   public void testWriteDate3() throws Exception {
     DateTime d = new DateTime(1991,10,11,23,10,DateTimeZone.forOffsetHours(-11));
@@ -154,6 +167,7 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest6", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       d.plusMinutes(1).toString(), FORMAT_4_DATE);
   }
+
   @Test
   public void testWriteDate2() throws Exception {
     DateTime d = new DateTime(1991,11,12,0,0, DateTimeZone.forID("US/Eastern"));
@@ -168,46 +182,48 @@ public class TestHCatStorer extends HCat
     pigValueRangeTestOverflow("junitTypeTest3", "date", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Throw,
       d.plusMinutes(1).toString(), FORMAT_4_DATE);
   }
+
   /**
-   * Note that the value that comes back from Hive will have local TZ on it.  Using local is 
+   * Note that the value that comes back from Hive will have local TZ on it.  Using local is
    * arbitrary but DateTime needs TZ (or will assume default) and Hive does not have TZ.
    * So if you start with Pig value in TZ=x and write to Hive, when you read it back the TZ may
    * be different.  The millis value should match, of course.
-   * 
+   *
    * @throws Exception
    */
   @Test
   public void testWriteTimestamp() throws Exception {
     DateTime d = new DateTime(1991,10,11,14,23,30, 10);//uses default TZ
-    pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(), 
+    pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(),
       d.toDateTime(DateTimeZone.getDefault()).toString());
     d = d.plusHours(2);
     pigValueRangeTest("junitTypeTest2", "timestamp", "datetime", HCatBaseStorer.OOR_VALUE_OPT_VALUES.Null,
       d.toString(), d.toDateTime(DateTimeZone.getDefault()).toString());
     d = d.toDateTime(DateTimeZone.UTC);
-    pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(), 
+    pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(),
       d.toDateTime(DateTimeZone.getDefault()).toString());
 
     d = new DateTime(1991,10,11,23,24,25, 26);
-    pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(), 
+    pigValueRangeTest("junitTypeTest1", "timestamp", "datetime", null, d.toString(),
       d.toDateTime(DateTimeZone.getDefault()).toString());
     d = d.toDateTime(DateTimeZone.UTC);
-    pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(), 
+    pigValueRangeTest("junitTypeTest3", "timestamp", "datetime", null, d.toString(),
       d.toDateTime(DateTimeZone.getDefault()).toString());
   }
   //End: tests that check values from Pig that are out of range for target column
 
-
   private void pigValueRangeTestOverflow(String tblName, String hiveType, String pigType,
     HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, String format) throws Exception {
     pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, null, format);
   }
+
   private void pigValueRangeTestOverflow(String tblName, String hiveType, String pigType,
                                  HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue) throws Exception {
     pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, null, null);
   }
+
   private void pigValueRangeTest(String tblName, String hiveType, String pigType,
-                                 HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, 
+                                 HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue,
                                  String expectedValue) throws Exception {
     pigValueRangeTest(tblName, hiveType, pigType, goal, inputValue, expectedValue, null);
   }
@@ -218,6 +234,7 @@ public class TestHCatStorer extends HCat
   String getStorageFormat() {
     return "RCFILE";
   }
+
   /**
    * This is used to test how Pig values of various data types which are out of range for Hive target
    * column are handled.  Currently the options are to raise an error or write NULL.
@@ -236,7 +253,7 @@ public class TestHCatStorer extends HCat
    * @param format date format to use for comparison of values since default DateTime.toString()
    *               includes TZ which is meaningless for Hive DATE type
    */
-  private void pigValueRangeTest(String tblName, String hiveType, String pigType, 
+  private void pigValueRangeTest(String tblName, String hiveType, String pigType,
                                  HCatBaseStorer.OOR_VALUE_OPT_VALUES goal, String inputValue, String expectedValue, String format)
     throws Exception {
     TestHCatLoader.dropTable(tblName, driver);
@@ -309,6 +326,7 @@ public class TestHCatStorer extends HCat
     Unfortunately Timestamp.toString() adjusts the value for local TZ and 't' is a String
     thus the timestamp in 't' doesn't match rawData*/
   }
+
   /**
    * Create a data file with datatypes added in 0.13.  Read it with Pig and use
    * Pig + HCatStorer to write to a Hive table.  Then read it using Pig and Hive
@@ -365,6 +383,7 @@ public class TestHCatStorer extends HCat
     }
     Assert.assertEquals("Expected " + NUM_ROWS + " rows; got " + numRowsRead + " file=" + INPUT_FILE_NAME, NUM_ROWS, numRowsRead);
   }
+
   static void dumpFile(String fileName) throws Exception {
     File file = new File(fileName);
     BufferedReader reader = new BufferedReader(new FileReader(file));
@@ -375,6 +394,7 @@ public class TestHCatStorer extends HCat
     }
     reader.close();
   }
+
   @Test
   public void testPartColsInData() throws IOException, CommandNeedRetryException {
 

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerMulti.java Fri Sep 19 04:42:59 2014
@@ -25,20 +25,26 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
 
-import junit.framework.TestCase;
-
 import org.apache.hadoop.fs.FileUtil;
 import org.apache.hadoop.hive.cli.CliSessionState;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
 import org.apache.hadoop.hive.ql.Driver;
 import org.apache.hadoop.hive.ql.session.SessionState;
+
 import org.apache.hive.hcatalog.common.HCatUtil;
 import org.apache.hive.hcatalog.data.Pair;
+
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
 
-public class TestHCatStorerMulti extends TestCase {
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestHCatStorerMulti {
   public static final String TEST_DATA_DIR = HCatUtil.makePathASafeFileName(
           System.getProperty("user.dir") + "/build/test/data/" +
                   TestHCatStorerMulti.class.getCanonicalName() + "-" + System.currentTimeMillis());
@@ -77,8 +83,8 @@ public class TestHCatStorerMulti extends
     createTable(tablename, schema, null);
   }
 
-  @Override
-  protected void setUp() throws Exception {
+  @Before
+  public void setUp() throws Exception {
     if (driver == null) {
       HiveConf hiveConf = new HiveConf(this.getClass());
       hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, "");
@@ -92,14 +98,13 @@ public class TestHCatStorerMulti extends
     cleanup();
   }
 
-  @Override
-  protected void tearDown() throws Exception {
+  @After
+  public void tearDown() throws Exception {
     cleanup();
   }
 
+  @Test
   public void testStoreBasicTable() throws Exception {
-
-
     createTable(BASIC_TABLE, "a int, b string");
 
     populateBasicFile();
@@ -117,6 +122,7 @@ public class TestHCatStorerMulti extends
     assertEquals(basicInputData.size(), unpartitionedTableValuesReadFromHiveDriver.size());
   }
 
+  @Test
   public void testStorePartitionedTable() throws Exception {
     createTable(PARTITIONED_TABLE, "a int, b string", "bkt string");
 
@@ -139,9 +145,8 @@ public class TestHCatStorerMulti extends
     assertEquals(basicInputData.size(), partitionedTableValuesReadFromHiveDriver.size());
   }
 
+  @Test
   public void testStoreTableMulti() throws Exception {
-
-
     createTable(BASIC_TABLE, "a int, b string");
     createTable(PARTITIONED_TABLE, "a int, b string", "bkt string");
 

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestHCatStorerWrapper.java Fri Sep 19 04:42:59 2014
@@ -25,10 +25,13 @@ import java.util.Iterator;
 import java.util.UUID;
 
 import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+
 import org.apache.hive.hcatalog.HcatTestUtils;
 import org.apache.hive.hcatalog.mapreduce.HCatBaseTest;
+
 import org.apache.pig.ExecType;
 import org.apache.pig.PigServer;
+
 import org.junit.Assert;
 import org.junit.Test;
 

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestOrcHCatStorer.java Fri Sep 19 04:42:59 2014
@@ -18,11 +18,15 @@
  */
 package org.apache.hive.hcatalog.pig;
 
-public class TestOrcHCatStorer extends TestHCatStorerMulti {
+import java.io.IOException;
 
-  @Override
-  protected String storageFormat() {
-    return "orc";
+import org.apache.hadoop.hive.ql.CommandNeedRetryException;
+
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestOrcHCatStorer extends TestHCatStorer {
+  @Override String getStorageFormat() {
+    return "ORC";
   }
 }
-

Modified: hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java (original)
+++ hive/branches/cbo/hcatalog/hcatalog-pig-adapter/src/test/java/org/apache/hive/hcatalog/pig/TestPigHCatUtil.java Fri Sep 19 04:42:59 2014
@@ -20,14 +20,18 @@
 package org.apache.hive.hcatalog.pig;
 
 import com.google.common.collect.Lists;
+
 import junit.framework.Assert;
+
 import org.apache.hive.hcatalog.common.HCatConstants;
 import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
 import org.apache.hive.hcatalog.data.schema.HCatSchema;
+
 import org.apache.pig.ResourceSchema;
 import org.apache.pig.ResourceSchema.ResourceFieldSchema;
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.util.UDFContext;
+
 import org.junit.Test;
 
 public class TestPigHCatUtil {

Modified: hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java (original)
+++ hive/branches/cbo/hcatalog/webhcat/svr/src/test/java/org/apache/hive/hcatalog/templeton/tool/TestTempletonUtils.java Fri Sep 19 04:42:59 2014
@@ -24,6 +24,7 @@ import java.net.URISyntaxException;
 import java.io.IOException;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.hive.shims.HadoopShimsSecure;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.util.StringUtils;
@@ -314,9 +315,9 @@ public class TestTempletonUtils {
 
   @Test
   public void testFindContainingJar() throws Exception {
-    String result = TempletonUtils.findContainingJar(ShimLoader.class, ".*hive-shims.*");
+    String result = TempletonUtils.findContainingJar(Configuration.class, ".*hadoop.*\\.jar.*");
     Assert.assertNotNull(result);
-    result = TempletonUtils.findContainingJar(HadoopShimsSecure.class, ".*hive-shims.*");
+    result = TempletonUtils.findContainingJar(FileSystem.class, ".*hadoop.*\\.jar.*");
     Assert.assertNotNull(result);
     result = TempletonUtils.findContainingJar(HadoopShimsSecure.class, ".*unknownjar.*");
     Assert.assertNull(result);

Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestJdbcDriver2.java Fri Sep 19 04:42:59 2014
@@ -735,7 +735,7 @@ public class TestJdbcDriver2 {
     assertTrue(res.next());
     // skip the last (partitioning) column since it is always non-null
     for (int i = 1; i < meta.getColumnCount(); i++) {
-      assertNull(res.getObject(i));
+      assertNull("Column " + i + " should be null", res.getObject(i));
     }
     // getXXX returns 0 for numeric types, false for boolean and null for other
     assertEquals(0, res.getInt(1));

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Fri Sep 19 04:42:59 2014
@@ -1145,7 +1145,10 @@ public class DDLSemanticAnalyzer extends
       }
     }
 
-    inputs.add(new ReadEntity(getTable(tableName)));
+    Table tbl = getTable(tableName, false);
+    if (tbl != null) {
+      inputs.add(new ReadEntity(getTable(tableName)));
+    }
 
     DropIndexDesc dropIdxDesc = new DropIndexDesc(indexName, tableName);
     rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(),

Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java Fri Sep 19 04:42:59 2014
@@ -129,7 +129,9 @@ public class UpdateDeleteSemanticAnalyze
     try {
       mTable = db.getTable(tableName[0], tableName[1]);
     } catch (HiveException e) {
-      throw new SemanticException(ErrorMsg.UPDATEDELETE_PARSE_ERROR.getMsg(), e);
+      LOG.error("Failed to find table " + getDotName(tableName) + " got exception " +
+          e.getMessage());
+      throw new SemanticException(ErrorMsg.INVALID_TABLE, getDotName(tableName));
     }
     List<FieldSchema> partCols = mTable.getPartCols();
 

Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/drop_index.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/drop_index.q?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/drop_index.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/drop_index.q Fri Sep 19 04:42:59 2014
@@ -1,2 +1,3 @@
 SET hive.exec.drop.ignorenonexistent=false;
 DROP INDEX IF EXISTS UnknownIndex ON src;
+DROP INDEX IF EXISTS UnknownIndex ON UnknownTable;

Modified: hive/branches/cbo/ql/src/test/results/clientpositive/drop_index.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/drop_index.q.out?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/drop_index.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/drop_index.q.out Fri Sep 19 04:42:59 2014
@@ -4,3 +4,7 @@ PREHOOK: Input: default@src
 POSTHOOK: query: DROP INDEX IF EXISTS UnknownIndex ON src
 POSTHOOK: type: DROPINDEX
 POSTHOOK: Input: default@src
+PREHOOK: query: DROP INDEX IF EXISTS UnknownIndex ON UnknownTable
+PREHOOK: type: DROPINDEX
+POSTHOOK: query: DROP INDEX IF EXISTS UnknownIndex ON UnknownTable
+POSTHOOK: type: DROPINDEX

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveChar.java Fri Sep 19 04:42:59 2014
@@ -55,14 +55,24 @@ public class LazyHiveChar extends
 
   @Override
   public void init(ByteArrayRef bytes, int start, int length) {
-    String byteData = null;
-    try {
-      byteData = Text.decode(bytes.getData(), start, length);
-      data.set(byteData, maxLength);
+    if (oi.isEscaped()) {
+      Text textData =  data.getTextValue();
+      // This is doing a lot of copying here, this could be improved by enforcing length
+      // at the same time as escaping rather than as separate steps.
+      LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length,
+          oi.getEscapeChar(),textData);
+      data.set(textData.toString(), maxLength);
       isNull = false;
-    } catch (CharacterCodingException e) {
-      isNull = true;
-      LOG.debug("Data not in the HiveChar data type range so converted to null.", e);
+    } else {
+      String byteData = null;
+      try {
+        byteData = Text.decode(bytes.getData(), start, length);
+        data.set(byteData, maxLength);
+        isNull = false;
+      } catch (CharacterCodingException e) {
+        isNull = true;
+        LOG.debug("Data not in the HiveChar data type range so converted to null.", e);
+      }
     }
   }
 

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyHiveVarchar.java Fri Sep 19 04:42:59 2014
@@ -55,14 +55,24 @@ public class LazyHiveVarchar extends
 
   @Override
   public void init(ByteArrayRef bytes, int start, int length) {
-    String byteData = null;
-    try {
-      byteData = Text.decode(bytes.getData(), start, length);
-      data.set(byteData, maxLength);
+    if (oi.isEscaped()) {
+      Text textData =  data.getTextValue();
+      // This is doing a lot of copying here, this could be improved by enforcing length
+      // at the same time as escaping rather than as separate steps.
+      LazyUtils.copyAndEscapeStringDataToText(bytes.getData(), start, length,
+          oi.getEscapeChar(),textData);
+      data.set(textData.toString(), maxLength);
       isNull = false;
-    } catch (CharacterCodingException e) {
-      isNull = true;
-      LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e);
+    } else {
+      try {
+        String byteData = null;
+        byteData = Text.decode(bytes.getData(), start, length);
+        data.set(byteData, maxLength);
+        isNull = false;
+      } catch (CharacterCodingException e) {
+        isNull = true;
+        LOG.debug("Data not in the HiveVarchar data type range so converted to null.", e);
+      }
     }
   }
 

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java Fri Sep 19 04:42:59 2014
@@ -40,40 +40,7 @@ public class LazyString extends LazyPrim
     if (oi.isEscaped()) {
       byte escapeChar = oi.getEscapeChar();
       byte[] inputBytes = bytes.getData();
-
-      // First calculate the length of the output string
-      int outputLength = 0;
-      for (int i = 0; i < length; i++) {
-        if (inputBytes[start + i] != escapeChar) {
-          outputLength++;
-        } else {
-          outputLength++;
-          i++;
-        }
-      }
-
-      // Copy the data over, so that the internal state of Text will be set to
-      // the required outputLength.
-      data.set(bytes.getData(), start, outputLength);
-
-      // We need to copy the data byte by byte only in case the
-      // "outputLength < length" (which means there is at least one escaped
-      // byte.
-      if (outputLength < length) {
-        int k = 0;
-        byte[] outputBytes = data.getBytes();
-        for (int i = 0; i < length; i++) {
-          byte b = inputBytes[start + i];
-          if (b != escapeChar || i == length - 1) {
-            outputBytes[k++] = b;
-          } else {
-            // get the next byte
-            i++;
-            outputBytes[k++] = inputBytes[start + i];
-          }
-        }
-        assert (k == outputLength);
-      }
+      LazyUtils.copyAndEscapeStringDataToText(inputBytes, start, length, escapeChar, data);
     } else {
       // if the data is not escaped, simply copy the data.
       data.set(bytes.getData(), start, length);

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Fri Sep 19 04:42:59 2014
@@ -412,6 +412,44 @@ public final class LazyUtils {
     }
   }
 
+  public static void copyAndEscapeStringDataToText(byte[] inputBytes, int start, int length,
+      byte escapeChar, Text data) {
+
+    // First calculate the length of the output string
+    int outputLength = 0;
+    for (int i = 0; i < length; i++) {
+      if (inputBytes[start + i] != escapeChar) {
+        outputLength++;
+      } else {
+        outputLength++;
+        i++;
+      }
+    }
+
+    // Copy the data over, so that the internal state of Text will be set to
+    // the required outputLength.
+    data.set(inputBytes, start, outputLength);
+
+    // We need to copy the data byte by byte only in case the
+    // "outputLength < length" (which means there is at least one escaped
+    // byte.
+    if (outputLength < length) {
+      int k = 0;
+      byte[] outputBytes = data.getBytes();
+      for (int i = 0; i < length; i++) {
+        byte b = inputBytes[start + i];
+        if (b != escapeChar || i == length - 1) {
+          outputBytes[k++] = b;
+        } else {
+          // get the next byte
+          i++;
+          outputBytes[k++] = inputBytes[start + i];
+        }
+      }
+      assert (k == outputLength);
+    }
+  }
+
   private LazyUtils() {
     // prevent instantiation
   }

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveCharObjectInspector.java Fri Sep 19 04:42:59 2014
@@ -29,12 +29,21 @@ public class LazyHiveCharObjectInspector
     extends AbstractPrimitiveLazyObjectInspector<HiveCharWritable>
     implements HiveCharObjectInspector {
 
+  private boolean escaped;
+  private byte escapeChar;
+
   // no-arg ctor required for Kyro
   public LazyHiveCharObjectInspector() {
   }
 
   public LazyHiveCharObjectInspector(CharTypeInfo typeInfo) {
+    this(typeInfo, false, (byte)0);
+  }
+
+  public LazyHiveCharObjectInspector(CharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
     super(typeInfo);
+    this.escaped = escaped;
+    this.escapeChar = escapeChar;
   }
 
   @Override
@@ -63,6 +72,14 @@ public class LazyHiveCharObjectInspector
     return ret;
   }
 
+  public boolean isEscaped() {
+    return escaped;
+  }
+
+  public byte getEscapeChar() {
+    return escapeChar;
+  }
+
   @Override
   public String toString() {
     return getTypeName();

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyHiveVarcharObjectInspector.java Fri Sep 19 04:42:59 2014
@@ -29,12 +29,21 @@ public class LazyHiveVarcharObjectInspec
     extends AbstractPrimitiveLazyObjectInspector<HiveVarcharWritable>
     implements HiveVarcharObjectInspector {
 
+  private boolean escaped;
+  private byte escapeChar;
+
   // no-arg ctor required for Kyro
   public LazyHiveVarcharObjectInspector() {
   }
 
   public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo) {
+    this(typeInfo, false, (byte)0);
+  }
+
+  public LazyHiveVarcharObjectInspector(VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
     super(typeInfo);
+    this.escaped = escaped;
+    this.escapeChar = escapeChar;
   }
 
   @Override
@@ -63,6 +72,14 @@ public class LazyHiveVarcharObjectInspec
     return ret;
   }
 
+  public boolean isEscaped() {
+    return escaped;
+  }
+
+  public byte getEscapeChar() {
+    return escapeChar;
+  }
+
   @Override
   public String toString() {
     return getTypeName();

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java Fri Sep 19 04:42:59 2014
@@ -78,8 +78,10 @@ public final class LazyPrimitiveObjectIn
     // prevent instantiation
   }
 
-  private static HashMap<ArrayList<Object>, LazyStringObjectInspector> cachedLazyStringObjectInspector =
-      new HashMap<ArrayList<Object>, LazyStringObjectInspector>();
+  // Lazy object inspectors for string/char/varchar will all be cached in the same map.
+  // Map key will be list of [typeInfo, isEscaped, escapeChar]
+  private static HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector> cachedLazyStringTypeOIs =
+      new HashMap<ArrayList<Object>, AbstractPrimitiveLazyObjectInspector>();
 
   private static Map<PrimitiveTypeInfo, AbstractPrimitiveLazyObjectInspector<?>>
      cachedPrimitiveLazyObjectInspectors =
@@ -121,6 +123,10 @@ public final class LazyPrimitiveObjectIn
     switch(primitiveCategory) {
     case STRING:
       return getLazyStringObjectInspector(escaped, escapeChar);
+    case CHAR:
+      return getLazyHiveCharObjectInspector((CharTypeInfo)typeInfo, escaped, escapeChar);
+    case VARCHAR:
+      return getLazyHiveVarcharObjectInspector((VarcharTypeInfo)typeInfo, escaped, escapeChar);
     case BOOLEAN:
       return getLazyBooleanObjectInspector(extBoolean);
     default:
@@ -157,13 +163,44 @@ public final class LazyPrimitiveObjectIn
 
   public static LazyStringObjectInspector getLazyStringObjectInspector(boolean escaped, byte escapeChar) {
     ArrayList<Object> signature = new ArrayList<Object>();
+    signature.add(TypeInfoFactory.stringTypeInfo);
     signature.add(Boolean.valueOf(escaped));
     signature.add(Byte.valueOf(escapeChar));
-    LazyStringObjectInspector result = cachedLazyStringObjectInspector
+    LazyStringObjectInspector result = (LazyStringObjectInspector) cachedLazyStringTypeOIs
         .get(signature);
     if (result == null) {
       result = new LazyStringObjectInspector(escaped, escapeChar);
-      cachedLazyStringObjectInspector.put(signature, result);
+      cachedLazyStringTypeOIs.put(signature, result);
+    }
+    return result;
+  }
+
+  public static LazyHiveCharObjectInspector getLazyHiveCharObjectInspector(
+      CharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
+    ArrayList<Object> signature = new ArrayList<Object>();
+    signature.add(typeInfo);
+    signature.add(Boolean.valueOf(escaped));
+    signature.add(Byte.valueOf(escapeChar));
+    LazyHiveCharObjectInspector result = (LazyHiveCharObjectInspector) cachedLazyStringTypeOIs
+        .get(signature);
+    if (result == null) {
+      result = new LazyHiveCharObjectInspector(typeInfo, escaped, escapeChar);
+      cachedLazyStringTypeOIs.put(signature, result);
+    }
+    return result;
+  }
+
+  public static LazyHiveVarcharObjectInspector getLazyHiveVarcharObjectInspector(
+      VarcharTypeInfo typeInfo, boolean escaped, byte escapeChar) {
+    ArrayList<Object> signature = new ArrayList<Object>();
+    signature.add(typeInfo);
+    signature.add(Boolean.valueOf(escaped));
+    signature.add(Byte.valueOf(escapeChar));
+    LazyHiveVarcharObjectInspector result = (LazyHiveVarcharObjectInspector) cachedLazyStringTypeOIs
+        .get(signature);
+    if (result == null) {
+      result = new LazyHiveVarcharObjectInspector(typeInfo, escaped, escapeChar);
+      cachedLazyStringTypeOIs.put(signature, result);
     }
     return result;
   }

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveCharObjectInspector.java Fri Sep 19 04:42:59 2014
@@ -91,6 +91,9 @@ public class WritableHiveCharObjectInspe
 
   @Override
   public Object set(Object o, HiveChar value) {
+    if (value == null) {
+      return null;
+    }
     HiveCharWritable writable = (HiveCharWritable) o;
     writable.set(value, getMaxLength());
     return o;
@@ -98,6 +101,9 @@ public class WritableHiveCharObjectInspe
 
   @Override
   public Object set(Object o, String value) {
+    if (value == null) {
+      return null;
+    }
     HiveCharWritable writable = (HiveCharWritable) o;
     writable.set(value, getMaxLength());
     return o;

Modified: hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java?rev=1626122&r1=1626121&r2=1626122&view=diff
==============================================================================
--- hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java (original)
+++ hive/branches/cbo/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableHiveVarcharObjectInspector.java Fri Sep 19 04:42:59 2014
@@ -96,6 +96,9 @@ implements SettableHiveVarcharObjectInsp
 
   @Override
   public Object set(Object o, HiveVarchar value) {
+    if (value == null) {
+      return null;
+    }
     HiveVarcharWritable writable = (HiveVarcharWritable)o;
     writable.set(value, getMaxLength());
     return o;
@@ -103,6 +106,9 @@ implements SettableHiveVarcharObjectInsp
 
   @Override
   public Object set(Object o, String value) {
+    if (value == null) {
+      return null;
+    }
     HiveVarcharWritable writable = (HiveVarcharWritable)o;
     writable.set(value, getMaxLength());
     return o;



Mime
View raw message