hadoop-hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From na...@apache.org
Subject svn commit: r790435 [1/3] - in /hadoop/hive/trunk: ./ ql/src/java/org/apache/hadoop/hive/ql/exec/ ql/src/java/org/apache/hadoop/hive/ql/parse/ ql/src/java/org/apache/hadoop/hive/ql/plan/ ql/src/test/org/apache/hadoop/hive/ql/io/ ql/src/test/queries/cli...
Date Thu, 02 Jul 2009 02:03:16 GMT
Author: namit
Date: Thu Jul  2 02:03:14 2009
New Revision: 790435

URL: http://svn.apache.org/viewvc?rev=790435&view=rev
Log:
HIVE-136. escaping in lazy serde
(Zheng Shao via namit)


Added:
    hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q
    hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyListObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyMapObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazyObjectInspectorFactory.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/LazySimpleStructObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/AbstractPrimitiveLazyObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyBooleanObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyByteObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyDoubleObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyFloatObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyIntObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyLongObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyPrimitiveObjectInspectorFactory.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyShortObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyStringObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/objectinspector/primitive/LazyVoidObjectInspector.java
Modified:
    hadoop/hive/trunk/CHANGES.txt
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
    hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java
    hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
    hadoop/hive/trunk/serde/if/serde.thrift
    hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java
    hadoop/hive/trunk/serde/src/gen-php/serde_constants.php
    hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyString.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyStruct.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazyListObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazyMapObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/LazySimpleStructObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/PrimitiveObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveJavaObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveWritableObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/JavaStringObjectInspector.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorUtils.java
    hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableStringObjectInspector.java
    hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyArrayMapStruct.java
    hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java
    hadoop/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazySimpleSerDe.java

Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Thu Jul  2 02:03:14 2009
@@ -56,6 +56,9 @@
     HIVE-567. jdbc: integrate hive with pentaho report designer
     (Raghotham Murthy via namit)
 
+    HIVE-136. escaping in lazy serde
+    (Zheng Shao via namit)
+
   IMPROVEMENTS
     HIVE-389. Option to build without ivy (jssarma)
 

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java Thu Jul  2 02:03:14 2009
@@ -31,6 +31,7 @@
 import java.util.Map;
 import java.util.SortedSet;
 import java.util.TreeSet;
+import java.util.Map.Entry;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -673,10 +674,10 @@
     if (crtTbl.getSerName() != null) {
       tbl.setSerializationLib(crtTbl.getSerName());
       if (crtTbl.getMapProp() != null) {
-        Iterator<Map.Entry<String, String>> iter = crtTbl.getMapProp()
+        Iterator<Entry<String, String>> iter = crtTbl.getMapProp()
             .entrySet().iterator();
         while (iter.hasNext()) {
-          Map.Entry<String, String> m = (Map.Entry<String, String>) iter.next();
+          Entry<String, String> m = (Entry<String, String>) iter.next();
           tbl.setSerdeParam(m.getKey(), m.getValue());
         }
       }
@@ -686,11 +687,13 @@
         tbl.setSerdeParam(Constants.SERIALIZATION_FORMAT, crtTbl
             .getFieldDelim());
       }
+      if (crtTbl.getFieldEscape() != null) {
+        tbl.setSerdeParam(Constants.ESCAPE_CHAR, crtTbl.getFieldEscape());
+      }
 
       if (crtTbl.getCollItemDelim() != null)
-        tbl
-            .setSerdeParam(Constants.COLLECTION_DELIM, crtTbl
-                .getCollItemDelim());
+        tbl.setSerdeParam(Constants.COLLECTION_DELIM, 
+            crtTbl.getCollItemDelim());
       if (crtTbl.getMapKeyDelim() != null)
         tbl.setSerdeParam(Constants.MAPKEY_DELIM, crtTbl.getMapKeyDelim());
       if (crtTbl.getLineDelim() != null)

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeIndexEvaluator.java Thu Jul  2 02:03:14 2009
@@ -78,7 +78,7 @@
     } else if (mainInspector.getCategory() == Category.MAP) {
       MapObjectInspector moi = (MapObjectInspector)mainInspector;
       Object indexObject;
-      if (((PrimitiveObjectInspector)moi.getMapKeyObjectInspector()).isWritable()) {
+      if (((PrimitiveObjectInspector)moi.getMapKeyObjectInspector()).preferWritable()) {
         indexObject = indexInspector.getPrimitiveWritableObject(index);
       } else {
         indexObject = indexInspector.getPrimitiveJavaObject(index);

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Thu Jul  2 02:03:14 2009
@@ -33,11 +33,11 @@
 import org.apache.hadoop.hive.ql.plan.aggregationDesc;
 import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
 import org.apache.hadoop.hive.ql.plan.groupByDesc;
-import org.apache.hadoop.hive.serde2.objectinspector.InspectableObject;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
@@ -68,6 +68,7 @@
   
   transient protected ExprNodeEvaluator[][] aggregationParameterFields;
   transient protected ObjectInspector[][] aggregationParameterObjectInspectors;
+  transient protected ObjectInspector[][] aggregationParameterStandardObjectInspectors;
   transient protected Object[][] aggregationParameterObjects;
   // In the future, we may allow both count(DISTINCT a) and sum(DISTINCT a) in the same SQL clause,
   // so aggregationIsDistinct is a boolean array instead of a single number. 
@@ -99,6 +100,11 @@
   transient long    numRowsCompareHashAggr;
   transient float   minReductionHashAggr;
 
+  // current Key ObjectInspectors are standard ObjectInspectors
+  transient protected ObjectInspector[] currentKeyObjectInspectors;
+  // new Key ObjectInspectors are objectInspectors from the parent
+  transient StructObjectInspector newKeyObjectInspector;
+  transient StructObjectInspector currentKeyObjectInspector;
   
   /**
    * This is used to store the position and field names for variable length fields.
@@ -149,18 +155,23 @@
     }
     newKeys = new ArrayList<Object>(keyFields.length);
     
+    currentKeyObjectInspectors = new ObjectInspector[conf.getKeys().size()];
+    
     // init aggregationParameterFields
     aggregationParameterFields = new ExprNodeEvaluator[conf.getAggregators().size()][];
     aggregationParameterObjectInspectors = new ObjectInspector[conf.getAggregators().size()][];
+    aggregationParameterStandardObjectInspectors = new ObjectInspector[conf.getAggregators().size()][];
     aggregationParameterObjects = new Object[conf.getAggregators().size()][];
     for (int i = 0; i < aggregationParameterFields.length; i++) {
       ArrayList<exprNodeDesc> parameters = conf.getAggregators().get(i).getParameters();
       aggregationParameterFields[i] = new ExprNodeEvaluator[parameters.size()];
       aggregationParameterObjectInspectors[i] = new ObjectInspector[parameters.size()];
+      aggregationParameterStandardObjectInspectors[i] = new ObjectInspector[parameters.size()];
       aggregationParameterObjects[i] = new Object[parameters.size()];
       for (int j = 0; j < parameters.size(); j++) {
         aggregationParameterFields[i][j] = ExprNodeEvaluatorFactory.get(parameters.get(j));
         aggregationParameterObjectInspectors[i][j] = null;
+        aggregationParameterStandardObjectInspectors[i][j] = null;
         aggregationParameterObjects[i][j] = null;
       }
     }
@@ -248,10 +259,22 @@
     for (int i = 0; i < keyFields.length; i++) {
       if (keyObjectInspectors[i] == null) {
         keyObjectInspectors[i] = keyFields[i].initialize(inputObjInspector[0]);
+        currentKeyObjectInspectors[i] = ObjectInspectorUtils.getStandardObjectInspector(keyObjectInspectors[i], 
+            ObjectInspectorCopyOption.WRITABLE);
       }
-      objectInspectors.set(i, keyObjectInspectors[i]);
+      objectInspectors.set(i, currentKeyObjectInspectors[i]);
     }
     
+    // Generate key names
+    ArrayList<String> keyNames = new ArrayList<String>(keyFields.length);
+    for (int i = 0; i < keyFields.length; i++) {
+      keyNames.add(fieldNames.get(i));
+    }
+    newKeyObjectInspector = 
+      ObjectInspectorFactory.getStandardStructObjectInspector(keyNames, Arrays.asList(keyObjectInspectors));
+    currentKeyObjectInspector = 
+      ObjectInspectorFactory.getStandardStructObjectInspector(keyNames, Arrays.asList(currentKeyObjectInspectors));
+    
     outputObjectInspector = 
       ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, objectInspectors);
 	
@@ -418,6 +441,8 @@
         if (oi == null) {
           oi = aggregationParameterFields[ai][pi].initialize(rowInspector);
           aggregationParameterObjectInspectors[ai][pi] = oi;
+          aggregationParameterStandardObjectInspectors[ai][pi]
+              = ObjectInspectorUtils.getStandardObjectInspector(oi, ObjectInspectorCopyOption.WRITABLE);
         }
         PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
         o[pi] = poi.getPrimitiveWritableObject(
@@ -457,7 +482,7 @@
             }
             for (int pi=0; pi<o.length; pi++) {
               lastInvoke[ai][pi] = ObjectInspectorUtils.copyToStandardObject(o[pi],
-                  aggregationParameterObjectInspectors[ai][pi]);
+                  aggregationParameterStandardObjectInspectors[ai][pi], ObjectInspectorCopyOption.WRITABLE);
             }
           }
         }
@@ -512,16 +537,18 @@
     }
   }
 
-  private static ArrayList<Object> deepCopyElements(Object[] keys, ObjectInspector[] keyObjectInspectors) {
+  private static ArrayList<Object> deepCopyElements(Object[] keys, ObjectInspector[] keyObjectInspectors,
+      ObjectInspectorCopyOption copyOption) {
     ArrayList<Object> result = new ArrayList<Object>(keys.length);
-    deepCopyElements(keys, keyObjectInspectors, result);
+    deepCopyElements(keys, keyObjectInspectors, result, copyOption);
     return result;
   }
   
-  private static void deepCopyElements(Object[] keys, ObjectInspector[] keyObjectInspectors, ArrayList<Object> result) {
+  private static void deepCopyElements(Object[] keys, ObjectInspector[] keyObjectInspectors, ArrayList<Object> result,
+      ObjectInspectorCopyOption copyOption) {
     result.clear();
     for (int i=0; i<keys.length; i++) {
-      result.add(ObjectInspectorUtils.copyToStandardObject(keys[i], keyObjectInspectors[i]));
+      result.add(ObjectInspectorUtils.copyToStandardObject(keys[i], keyObjectInspectors[i], copyOption));
     }
   }
   
@@ -531,10 +558,11 @@
     boolean newEntry = false;
 
     // hash-based aggregations
-    aggs = hashAggregations.get(newKeys);
+    ArrayList<Object> newDefaultKeys = deepCopyElements(keyObjects, keyObjectInspectors, ObjectInspectorCopyOption.WRITABLE);
+    aggs = hashAggregations.get(newDefaultKeys);
     if (aggs == null) {
       aggs = newAggregations();
-      hashAggregations.put(deepCopyElements(keyObjects, keyObjectInspectors), aggs);
+      hashAggregations.put(newDefaultKeys, aggs);
       newEntry = true;
       numRowsHashTbl++;      // new entry in the hash table
     }
@@ -552,7 +580,9 @@
     // Prepare aggs for updating
     UDAFEvaluator[] aggs = null;
     Object[][] lastInvoke = null;
-    boolean keysAreEqual = newKeys.equals(currentKeys);
+    boolean keysAreEqual = ObjectInspectorUtils.compare(
+        newKeys, newKeyObjectInspector,
+        currentKeys, currentKeyObjectInspector) == 0;
     
     // Forward the current keys if needed for sort-based aggregation
     if (currentKeys != null && !keysAreEqual)
@@ -563,7 +593,7 @@
       if (currentKeys == null) {
         currentKeys = new ArrayList<Object>(keyFields.length);
       }
-      deepCopyElements(keyObjects, keyObjectInspectors, currentKeys);
+      deepCopyElements(keyObjects, keyObjectInspectors, currentKeys, ObjectInspectorCopyOption.WRITABLE);
       
       // Init aggregations
       for(UDAFEvaluator aggregation: aggregations)
@@ -596,7 +626,7 @@
         if (key != null) {
           if (key instanceof String) {
             totalVariableSize += ((String)key).length();
-          } else {
+          } else if (key instanceof Text) {
             totalVariableSize += ((Text)key).getLength();
           }
         }

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java Thu Jul  2 02:03:14 2009
@@ -155,6 +155,7 @@
     List<Order>       sortCols      = null;
     int               numBuckets    = -1;
     String            fieldDelim    = null;
+    String            fieldEscape   = null;
     String            collItemDelim = null;
     String            mapKeyDelim   = null;
     String            lineDelim     = null;
@@ -216,6 +217,9 @@
             switch (rowChild.getToken().getType()) {
               case HiveParser.TOK_TABLEROWFORMATFIELD:
                 fieldDelim = unescapeSQLString(rowChild.getChild(0).getText());
+                if (rowChild.getChildCount()>=2) {
+                  fieldEscape = unescapeSQLString(rowChild.getChild(1).getText());
+                }
                 break;
               case HiveParser.TOK_TABLEROWFORMATCOLLITEMS:
                 collItemDelim = unescapeSQLString(rowChild.getChild(0).getText());
@@ -269,7 +273,8 @@
       createTableDesc crtTblDesc = 
         new createTableDesc(tableName, isExt, cols, partCols, bucketCols, 
                             sortCols, numBuckets,
-                            fieldDelim, collItemDelim, mapKeyDelim, lineDelim,
+                            fieldDelim, fieldEscape,
+                            collItemDelim, mapKeyDelim, lineDelim,
                             comment, inputFormat, outputFormat, location, serde, 
                             mapProp, ifNotExists);
   

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g Thu Jul  2 02:03:14 2009
@@ -383,8 +383,8 @@
 @init { msgs.push("table row format's field separator"); }
 @after { msgs.pop(); }
     :
-      KW_FIELDS KW_TERMINATED KW_BY fldIdnt=StringLiteral 
-    -> ^(TOK_TABLEROWFORMATFIELD $fldIdnt)
+      KW_FIELDS KW_TERMINATED KW_BY fldIdnt=StringLiteral (KW_ESCAPED KW_BY fldEscape=StringLiteral)?
+    -> ^(TOK_TABLEROWFORMATFIELD $fldIdnt $fldEscape?)
     ;
 
 tableRowFormatCollItemsIdentifier
@@ -1185,6 +1185,7 @@
 KW_DELIMITED: 'DELIMITED';
 KW_FIELDS: 'FIELDS';
 KW_TERMINATED: 'TERMINATED';
+KW_ESCAPED: 'ESCAPED';
 KW_COLLECTION: 'COLLECTION';
 KW_ITEMS: 'ITEMS';
 KW_KEYS: 'KEYS';

Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/createTableDesc.java Thu Jul  2 02:03:14 2009
@@ -38,6 +38,7 @@
   List<Order>         sortCols;
   int                 numBuckets;
   String              fieldDelim;
+  String              fieldEscape;
   String              collItemDelim;
   String              mapKeyDelim;
   String              lineDelim;
@@ -53,6 +54,7 @@
                          List<FieldSchema> cols, List<FieldSchema> partCols,
                          List<String> bucketCols, List<Order> sortCols, 
                          int numBuckets, String fieldDelim, 
+                         String fieldEscape,
                          String collItemDelim,
                          String mapKeyDelim, String lineDelim, 
                          String comment, String inputFormat, String outputFormat,
@@ -66,6 +68,7 @@
     this.cols            = cols;
     this.comment         = comment;
     this.fieldDelim      = fieldDelim;
+    this.fieldEscape     = fieldEscape;
     this.inputFormat     = inputFormat;
     this.outputFormat    = outputFormat;
     this.lineDelim       = lineDelim;
@@ -149,6 +152,15 @@
     this.fieldDelim = fieldDelim;
   }
 
+  @explain(displayName="field escape")
+  public String getFieldEscape() {
+    return fieldEscape;
+  }
+
+  public void setFieldEscape(String fieldEscape) {
+    this.fieldEscape = fieldEscape;
+  }
+
   @explain(displayName="collection delimiter")
   public String getCollItemDelim() {
     return collItemDelim;

Modified: hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java (original)
+++ hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/io/TestRCFile.java Thu Jul  2 02:03:14 2009
@@ -38,8 +38,10 @@
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
 import org.apache.hadoop.io.IntWritable;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
@@ -176,10 +178,12 @@
       assertEquals(8, fieldRefs.size());
       for (int j = 0; j < fieldRefs.size(); j++) {
         Object fieldData = oi.getStructFieldData(row, fieldRefs.get(j));
+        Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData, 
+            fieldRefs.get(j).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE);
         if (i == 0)
-          assertEquals("Field " + i, fieldData, expectedRecord_1[j]);
+          assertEquals("Field " + i, standardWritableData, expectedRecord_1[j]);
         else
-          assertEquals("Field " + i, fieldData, expectedRecord_2[j]);
+          assertEquals("Field " + i, standardWritableData, expectedRecord_2[j]);
       }
     }
 
@@ -300,7 +304,9 @@
       assertEquals(8, fieldRefs.size());
       for (int i = 0; i < fieldRefs.size(); i++) {
         Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
-        assertEquals("Field " + i, fieldData, expectedFieldsData[i]);
+        Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData, 
+            fieldRefs.get(i).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE);
+        assertEquals("Field " + i, standardWritableData, expectedFieldsData[i]);
       }
       // Serialize
       assertEquals(BytesRefArrayWritable.class, serDe.getSerializedClass());
@@ -336,7 +342,9 @@
 
       for (int i = 0; i < fieldRefs.size(); i++) {
         Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));
-        assertEquals("Field " + i, fieldData, expectedPartitalFieldsData[i]);
+        Object standardWritableData = ObjectInspectorUtils.copyToStandardObject(fieldData, 
+            fieldRefs.get(i).getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE);
+        assertEquals("Field " + i, standardWritableData, expectedPartitalFieldsData[i]);
       }
 
       assertEquals(BytesRefArrayWritable.class, serDe.getSerializedClass());

Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q?rev=790435&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/create_escape.q Thu Jul  2 02:03:14 2009
@@ -0,0 +1,16 @@
+DROP TABLE table1;
+DROP TABLE table2;
+DROP TABLE table3;
+
+CREATE TABLE table1 (a STRING, b STRING)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\\'
+STORED AS TEXTFILE;
+
+DESCRIBE table1;
+DESCRIBE EXTENDED table1;
+
+INSERT OVERWRITE TABLE table1 SELECT key, '\\\t\\' FROM src WHERE key = 86;
+
+SELECT * FROM table1;
+
+DROP TABLE table1;

Added: hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out?rev=790435&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out (added)
+++ hadoop/hive/trunk/ql/src/test/results/clientpositive/create_escape.q.out Thu Jul  2 02:03:14 2009
@@ -0,0 +1,22 @@
+query: DROP TABLE table1
+query: DROP TABLE table2
+query: DROP TABLE table3
+query: CREATE TABLE table1 (a STRING, b STRING)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\\'
+STORED AS TEXTFILE
+query: DESCRIBE table1
+a	string	
+b	string	
+query: DESCRIBE EXTENDED table1
+a	string	
+b	string	
+	 	 
+Detailed Table Information	Table(tableName:table1,dbName:default,owner:zshao,createTime:1245330503,lastAccessTime:0,retention:0,sd:StorageDescriptor(cols:[FieldSchema(name:a,type:string,comment:null), FieldSchema(name:b,type:string,comment:null)],location:file:/data/users/zshao/tools/136-trunk-apache-hive/build/ql/test/data/warehouse/table1,inputFormat:org.apache.hadoop.mapred.TextInputFormat,outputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat,compressed:false,numBuckets:-1,serdeInfo:SerDeInfo(name:null,serializationLib:org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe,parameters:{serialization.format=9,escape.delim=\,field.delim=	}),bucketCols:[],sortCols:[],parameters:{}),partitionKeys:[],parameters:{})
+query: INSERT OVERWRITE TABLE table1 SELECT key, '\\\t\\' FROM src WHERE key = 86
+Input: default/src
+Output: default/table1
+query: SELECT * FROM table1
+Input: default/table1
+Output: file:/data/users/zshao/tools/136-trunk-apache-hive/build/ql/tmp/822936101/10000
+86	\	\
+query: DROP TABLE table1

Modified: hadoop/hive/trunk/serde/if/serde.thrift
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/if/serde.thrift?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/if/serde.thrift (original)
+++ hadoop/hive/trunk/serde/if/serde.thrift Thu Jul  2 02:03:14 2009
@@ -17,6 +17,7 @@
 const string LINE_DELIM = "line.delim"
 const string MAPKEY_DELIM = "mapkey.delim"
 const string QUOTE_CHAR = "quote.delim"
+const string ESCAPE_CHAR = "escape.delim"
 
 typedef string PrimitiveType
 typedef string CollectionType

Modified: hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java (original)
+++ hadoop/hive/trunk/serde/src/gen-java/org/apache/hadoop/hive/serde/Constants.java Thu Jul  2 02:03:14 2009
@@ -39,6 +39,8 @@
 
   public static final String QUOTE_CHAR = "quote.delim";
 
+  public static final String ESCAPE_CHAR = "escape.delim";
+
   public static final String VOID_TYPE_NAME = "void";
 
   public static final String BOOLEAN_TYPE_NAME = "boolean";

Modified: hadoop/hive/trunk/serde/src/gen-php/serde_constants.php
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-php/serde_constants.php?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-php/serde_constants.php (original)
+++ hadoop/hive/trunk/serde/src/gen-php/serde_constants.php Thu Jul  2 02:03:14 2009
@@ -32,6 +32,8 @@
 
 $GLOBALS['serde_CONSTANTS']['QUOTE_CHAR'] = 'quote.delim';
 
+$GLOBALS['serde_CONSTANTS']['ESCAPE_CHAR'] = 'escape.delim';
+
 $GLOBALS['serde_CONSTANTS']['VOID_TYPE_NAME'] = 'void';
 
 $GLOBALS['serde_CONSTANTS']['BOOLEAN_TYPE_NAME'] = 'boolean';

Modified: hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py (original)
+++ hadoop/hive/trunk/serde/src/gen-py/org_apache_hadoop_hive_serde/constants.py Thu Jul  2 02:03:14 2009
@@ -31,6 +31,8 @@
 
 QUOTE_CHAR = 'quote.delim'
 
+ESCAPE_CHAR = 'escape.delim'
+
 VOID_TYPE_NAME = 'void'
 
 BOOLEAN_TYPE_NAME = 'boolean'

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java Thu Jul  2 02:03:14 2009
@@ -25,9 +25,18 @@
 import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.StructField;
 import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
 import org.apache.hadoop.io.Text;
 
 public class SerDeUtils {
@@ -96,6 +105,9 @@
     return true;
   }
 
+  /**
+   * Escape a String in JSON format.
+   */
   public static String escapeString(String str) {
     int length = str.length();
     StringBuilder escape = new StringBuilder(length + 16);
@@ -181,7 +193,7 @@
     return sb.toString();
   }
 
-  
+
   static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) {
 
     switch(oi.getCategory()) {
@@ -189,17 +201,46 @@
         PrimitiveObjectInspector poi = (PrimitiveObjectInspector)oi;
         if (o == null) {
           sb.append("null");
-        } else if (oi.getTypeName().equals(Constants.STRING_TYPE_NAME)) {
-          String s = (String)poi.getPrimitiveJavaObject(o);
-          sb.append(QUOTE);
-          sb.append(escapeString(s));
-          sb.append(QUOTE);
-        } else if (oi.getTypeName().equals(Constants.BOOLEAN_TYPE_NAME)) {
-          Boolean b = (Boolean)poi.getPrimitiveJavaObject(o);
-          sb.append(b.booleanValue() ? "True" : "False");
         } else {
-          // it's a number - so doesn't need to be escaped.
-          sb.append(o.toString());
+          switch (poi.getPrimitiveCategory()) {
+          case BOOLEAN: {
+            boolean b = ((BooleanObjectInspector)poi).get(o);
+            sb.append(b ? "true" : "false");
+            break;
+          }
+          case BYTE: {
+            sb.append(((ByteObjectInspector)poi).get(o));
+            break;
+          }
+          case SHORT: {
+            sb.append(((ShortObjectInspector)poi).get(o));
+            break;
+          }
+          case INT: {
+            sb.append(((IntObjectInspector)poi).get(o));
+            break;
+          }
+          case LONG: {
+            sb.append(((LongObjectInspector)poi).get(o));
+            break;
+          }
+          case FLOAT: {
+            sb.append(((FloatObjectInspector)poi).get(o));
+            break;
+          }
+          case DOUBLE: {
+            sb.append(((DoubleObjectInspector)poi).get(o));
+            break;
+          }
+          case STRING: {
+            sb.append('"'); 
+            sb.append(escapeString(((StringObjectInspector)poi).getPrimitiveJavaObject(o)));
+            sb.append('"'); 
+            break;
+          }
+          default:
+            throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
+          }
         }
         break;
       }

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java Thu Jul  2 02:03:14 2009
@@ -84,13 +84,15 @@
     serdeParams = LazySimpleSerDe.initSerdeParams(job, tbl, getClass()
         .getName());
 
-    cachedLazyStruct = new ColumnarStruct(serdeParams.getRowTypeInfo());
     // Create the ObjectInspectors for the fields. Note: Currently
     // ColumnarObject uses same ObjectInpector as LazyStruct
     cachedObjectInspector = LazyFactory.createColumnarStructInspector(
         serdeParams.getColumnNames(), serdeParams.getColumnTypes(), serdeParams
-            .getSeparators(), serdeParams.getNullSequence());
+            .getSeparators(), serdeParams.getNullSequence(), serdeParams.isEscaped(),
+            serdeParams.getEscapeChar());
 
+    cachedLazyStruct = new ColumnarStruct(cachedObjectInspector);
+    
     int size = serdeParams.getColumnTypes().size();
     field = new BytesRefWritable[size];
     for (int i = 0; i < size; i++) {
@@ -198,13 +200,17 @@
             && (declaredFields == null || declaredFields.get(i)
                 .getFieldObjectInspector().getCategory().equals(
                     Category.PRIMITIVE))) {
-          LazySimpleSerDe.serialize(serializeStream, SerDeUtils.getJSONString(
-              f, foi),
+          LazySimpleSerDe.serialize(serializeStream, 
+              SerDeUtils.getJSONString(f, foi),
               PrimitiveObjectInspectorFactory.javaStringObjectInspector,
-              serdeParams.getSeparators(), 1, serdeParams.getNullSequence());
+              serdeParams.getSeparators(), 1, serdeParams.getNullSequence(),
+              serdeParams.isEscaped(), serdeParams.getEscapeChar(),
+              serdeParams.getNeedsEscape());
         } else {
-          LazySimpleSerDe.serialize(serializeStream, f, foi, serdeParams
-              .getSeparators(), 1, serdeParams.getNullSequence());
+          LazySimpleSerDe.serialize(serializeStream, f, foi, 
+              serdeParams.getSeparators(), 1, serdeParams.getNullSequence(),
+              serdeParams.isEscaped(), serdeParams.getEscapeChar(),
+              serdeParams.getNeedsEscape());
         }
 
         field[i].set(serializeStream.getData(), count, serializeStream

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java Thu Jul  2 02:03:14 2009
@@ -25,8 +25,9 @@
 import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
 import org.apache.hadoop.hive.serde2.lazy.LazyObject;
 import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.io.Text;
 
 /**
@@ -39,17 +40,6 @@
 public class ColumnarStruct {
 
   /**
-   * The TypeInfo for this LazyNonPrimitive.
-   */
-  TypeInfo typeInfo;
-  List<TypeInfo> fieldTypeInfos;
-
-  /**
-   * Whether the data is already parsed or not.
-   */
-  boolean parsed;
-
-  /**
    * The fields of the struct.
    */
   LazyObject[] fields;
@@ -58,18 +48,17 @@
    * Construct a ColumnarStruct object with the TypeInfo. It creates the first
    * level object at the first place
    * 
-   * @param typeInfo
-   *          the TypeInfo representing the type of this LazyStruct.
+   * @param oi
+   *          the ObjectInspector representing the type of this LazyStruct.
    */
-  public ColumnarStruct(TypeInfo typeInfo) {
-    this.typeInfo = typeInfo;
-    fieldTypeInfos = ((StructTypeInfo) typeInfo).getAllStructFieldTypeInfos();
-    int num = fieldTypeInfos.size();
+  public ColumnarStruct(ObjectInspector oi) {
+    List<? extends StructField> fieldRefs = ((StructObjectInspector) oi).getAllStructFieldRefs();
+    int num = fieldRefs.size();
     fields = new LazyObject[num];
     cachedByteArrayRef = new ByteArrayRef[num];
     fieldIsNull = new boolean[num];
     for (int i = 0; i < num; i++) {
-      fields[i] = LazyFactory.createLazyObject(fieldTypeInfos.get(i));
+      fields[i] = LazyFactory.createLazyObject(fieldRefs.get(i).getFieldObjectInspector());
       cachedByteArrayRef[i] = new ByteArrayRef();
       fieldIsNull[i] = false;
     }
@@ -147,8 +136,6 @@
     }
     for (; fieldIndex < fields.length; fieldIndex++)
       fieldIsNull[fieldIndex] = true;
-
-    parsed = true;
   }
 
   ArrayList<Object> cachedList;
@@ -171,4 +158,4 @@
     }
     return cachedList;
   }
-}
\ No newline at end of file
+}

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyArray.java Thu Jul  2 02:03:14 2009
@@ -21,17 +21,17 @@
 import java.util.Arrays;
 import java.util.List;
 
-import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
 import org.apache.hadoop.io.Text;
 
 /**
  * LazyArray stores an array of Lazy Objects.
  * 
  * LazyArray does not deal with the case of a NULL array. That is handled
- * by LazyArrayObjectInspector.
+ * by the parent LazyObject.
  */
-public class LazyArray extends LazyNonPrimitive {
+public class LazyArray extends LazyNonPrimitive<LazyListObjectInspector> {
 
   /**
    * Whether the data is already parsed or not.
@@ -65,11 +65,12 @@
   LazyObject[] arrayElements;
 
   /**
-   * Construct a LazyArray object with the TypeInfo.
-   * @param typeInfo  the TypeInfo representing the type of this LazyArray.
+   * Construct a LazyArray object with the ObjectInspector.
+   * @param oi  the oi representing the type of this LazyArray as well as meta 
+   *            information like separator etc.
    */
-  protected LazyArray(TypeInfo typeInfo) {
-    super(typeInfo);
+  protected LazyArray(LazyListObjectInspector oi) {
+    super(oi);
   }
 
   /**
@@ -86,7 +87,7 @@
    * Enlarge the size of arrays storing information for the elements inside 
    * the array.
    */
-  protected void enlargeArrays() {
+  private void enlargeArrays() {
     if (startPosition == null) {
       int initialSize = 2;
       startPosition = new int[initialSize]; 
@@ -102,9 +103,13 @@
   /**
    * Parse the bytes and fill arrayLength and startPosition.
    */
-  private void parse(byte separator, Text nullSequence) {
+  private void parse() {
     parsed = true;
     
+    byte separator = oi.getSeparator();
+    boolean isEscaped = oi.isEscaped();
+    byte escapeChar = oi.getEscapeChar();
+    
     // empty array?
     if (length == 0) {
       arrayLength = 0;
@@ -131,7 +136,14 @@
         arrayLength++;
         elementByteBegin = elementByteEnd + 1;
       }
-      elementByteEnd++;
+      if (isEscaped && bytes[elementByteEnd] == escapeChar
+          && elementByteEnd+1 < arrayByteEnd) {
+        // ignore the char after escape_char
+        elementByteEnd += 2;
+      } else {
+        elementByteEnd ++;
+      }
+      
     }
     // Store arrayByteEnd+1 in startPosition[arrayLength]
     // so that we can use the same formula to compute the length of
@@ -148,21 +160,22 @@
    * Returns the actual primitive object at the index position
    * inside the array represented by this LazyObject.
    */
-  public Object getListElementObject(int index, byte separator, 
-      Text nullSequence) {
+  public Object getListElementObject(int index) {
     if (!parsed) {
-      parse(separator, nullSequence);
+      parse();
     }
     if (index < 0 || index >= arrayLength) {
       return null;
     }
-    return uncheckedGetElement(index, nullSequence);
+    return uncheckedGetElement(index);
   }
   
   /**
-   * Get the element without checking parsed or out-of-bound index.
+   * Get the element without checking out-of-bound index.
    */
-  private Object uncheckedGetElement(int index, Text nullSequence) {
+  private Object uncheckedGetElement(int index) {
+    Text nullSequence = oi.getNullSequence();
+
     int elementLength = startPosition[index+1] - startPosition[index] - 1;
     if (elementLength == nullSequence.getLength() 
         && 0 == LazyUtils.compare(bytes.getData(), startPosition[index], 
@@ -174,7 +187,7 @@
         elementInited[index] = true;
         if (arrayElements[index] == null) {
           arrayElements[index] = LazyFactory.createLazyObject(
-            ((ListTypeInfo)typeInfo).getListElementTypeInfo());
+            ((ListObjectInspector)oi).getListElementObjectInspector());
         }
         arrayElements[index].init(bytes, startPosition[index], 
             elementLength);
@@ -185,9 +198,9 @@
   
   /** Returns -1 for null array.
    */
-  public int getListLength(byte separator, Text nullSequence) {
+  public int getListLength() {
     if (!parsed) {
-      parse(separator, nullSequence);
+      parse();
     }
     return arrayLength;
   }
@@ -200,9 +213,9 @@
   /** Returns the List of actual primitive objects.
    *  Returns null for null array.
    */
-  public List<Object> getList(byte separator, Text nullSequence) {
+  public List<Object> getList() {
     if (!parsed) {
-      parse(separator, nullSequence);
+      parse();
     }
     if (arrayLength == -1) {
       return null;
@@ -213,7 +226,7 @@
       cachedList.clear();
     }
     for (int index=0; index<arrayLength; index++) {
-      cachedList.add(uncheckedGetElement(index, nullSequence));
+      cachedList.add(uncheckedGetElement(index));
     }
     return cachedList;
   }

Added: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java?rev=790435&view=auto
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java (added)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java Thu Jul  2 02:03:14 2009
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.serde2.lazy;
+
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector;
+import org.apache.hadoop.io.BooleanWritable;
+
+/**
+ * LazyObject for storing a value of boolean.
+ * 
+ * <p>
+ * Part of the code is adapted from Apache Harmony Project.
+ * 
+ * As with the specification, this implementation relied on code laid out in <a
+ * href="http://www.hackersdelight.org/">Henry S. Warren, Jr.'s Hacker's
+ * Delight, (Addison Wesley, 2002)</a> as well as <a
+ * href="http://aggregate.org/MAGIC/">The Aggregate's Magic Algorithms</a>.
+ * </p>
+ * 
+ */
+public class LazyBoolean extends LazyPrimitive<LazyBooleanObjectInspector, BooleanWritable> {
+
+  public LazyBoolean(LazyBooleanObjectInspector oi) {
+    super(oi);
+    data = new BooleanWritable();
+  }
+
+  public LazyBoolean(LazyBoolean copy) {
+    super(copy);
+    data = new BooleanWritable(copy.data.get());
+  }
+  
+  @Override
+  public void init(ByteArrayRef bytes, int start, int length) {
+    if (length == 4 
+        && Character.toUpperCase(bytes.getData()[start]) == 'T'
+        && Character.toUpperCase(bytes.getData()[start+1]) == 'R'
+        && Character.toUpperCase(bytes.getData()[start+2]) == 'U'
+        && Character.toUpperCase(bytes.getData()[start+3]) == 'E') {
+      data.set(true);
+      isNull = false;
+    } else if (length == 5
+          && Character.toUpperCase(bytes.getData()[start]) == 'F'
+          && Character.toUpperCase(bytes.getData()[start+1]) == 'A'
+          && Character.toUpperCase(bytes.getData()[start+2]) == 'L'
+          && Character.toUpperCase(bytes.getData()[start+3]) == 'S'
+          && Character.toUpperCase(bytes.getData()[start+4]) == 'E') {
+      data.set(false);
+      isNull = false;
+    } else { 
+      isNull = true;
+    }
+  }
+  
+
+}

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java Thu Jul  2 02:03:14 2009
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.serde2.lazy;
 
 import org.apache.hadoop.hive.serde2.io.ByteWritable;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector;
 
 /**
  * LazyObject for storing a value of Byte.
@@ -32,12 +33,18 @@
  * </p>
  * 
  */
-public class LazyByte extends LazyPrimitive<ByteWritable> {
+public class LazyByte extends LazyPrimitive<LazyByteObjectInspector, ByteWritable> {
 
-  public LazyByte() {
+  public LazyByte(LazyByteObjectInspector oi) {
+    super(oi);
     data = new ByteWritable();
   }
 
+  public LazyByte(LazyByte copy) {
+    super(copy);
+    data = new ByteWritable(copy.data.get());
+  }
+  
   @Override
   public void init(ByteArrayRef bytes, int start, int length) {
     try {

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java Thu Jul  2 02:03:14 2009
@@ -20,17 +20,24 @@
 import java.nio.charset.CharacterCodingException;
 
 import org.apache.hadoop.hive.serde2.io.DoubleWritable;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector;
 import org.apache.hadoop.io.Text;
 
 /**
  * LazyObject for storing a value of Double.
  * 
  */
-public class LazyDouble extends LazyPrimitive<DoubleWritable> {
+public class LazyDouble extends LazyPrimitive<LazyDoubleObjectInspector, DoubleWritable> {
 
-  public LazyDouble() {
+  public LazyDouble(LazyDoubleObjectInspector oi) {
+    super(oi);
     data = new DoubleWritable();
   }
+
+  public LazyDouble(LazyDouble copy) {
+    super(copy);
+    data = new DoubleWritable(copy.data.get());
+  }  
   
   @Override
   public void init(ByteArrayRef bytes, int start, int length) {

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java Thu Jul  2 02:03:14 2009
@@ -20,11 +20,32 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyListObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazySimpleStructObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyByteObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyIntObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyLongObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyPrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyShortObjectInspector;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyStringObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableDoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableFloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableIntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableLongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
 import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
@@ -37,29 +58,32 @@
   /**
    * Create a lazy primitive class given the type name. 
    */
-  public static LazyPrimitive<?> createLazyPrimitiveClass(String typeName) {
-    PrimitiveCategory p = PrimitiveObjectInspectorUtils.getTypeEntryFromTypeName(typeName).primitiveCategory;
+  public static LazyPrimitive<?,?> createLazyPrimitiveClass(PrimitiveObjectInspector oi) {
+    PrimitiveCategory p = oi.getPrimitiveCategory();
     switch(p) {
+      case BOOLEAN: {
+        return new LazyBoolean((LazyBooleanObjectInspector)oi);
+      }
       case BYTE: {
-        return new LazyByte();
+        return new LazyByte((LazyByteObjectInspector)oi);
       }
       case SHORT: {
-        return new LazyShort();
+        return new LazyShort((LazyShortObjectInspector)oi);
       }
       case INT: {
-        return new LazyInteger();
+        return new LazyInteger((LazyIntObjectInspector)oi);
       }
       case LONG: {
-        return new LazyLong();
+        return new LazyLong((LazyLongObjectInspector)oi);
       }
       case FLOAT: {
-        return new LazyFloat();
+        return new LazyFloat((LazyFloatObjectInspector)oi);
       }
       case DOUBLE: {
-        return new LazyDouble();
+        return new LazyDouble((LazyDoubleObjectInspector)oi);
       }
       case STRING: {
-        return new LazyString();
+        return new LazyString((LazyStringObjectInspector)oi);
       }
       default: {
         throw new RuntimeException("Internal error: no LazyObject for " + p);
@@ -70,17 +94,17 @@
   /**
    * Create a hierarchical LazyObject based on the given typeInfo.
    */
-  public static LazyObject createLazyObject(TypeInfo typeInfo) {
-    ObjectInspector.Category c = typeInfo.getCategory();
+  public static LazyObject createLazyObject(ObjectInspector oi) {
+    ObjectInspector.Category c = oi.getCategory();
     switch(c) {
     case PRIMITIVE:
-      return createLazyPrimitiveClass(typeInfo.getTypeName());
+      return createLazyPrimitiveClass((PrimitiveObjectInspector)oi);
     case MAP:
-      return new LazyMap(typeInfo);      
+      return new LazyMap((LazyMapObjectInspector)oi);      
     case LIST: 
-      return new LazyArray(typeInfo);      
+      return new LazyArray((LazyListObjectInspector)oi);      
     case STRUCT:
-      return new LazyStruct(typeInfo);      
+      return new LazyStruct((LazySimpleStructObjectInspector)oi);      
     }
 
     throw new RuntimeException("Hive LazySerDe Internal error.");
@@ -99,24 +123,27 @@
    * @return  The ObjectInspector
    */
   public static ObjectInspector createLazyObjectInspector(TypeInfo typeInfo, byte[] separator, 
-      int separatorIndex, Text nullSequence) {
+      int separatorIndex, Text nullSequence, boolean escaped, byte escapeChar) {
     ObjectInspector.Category c = typeInfo.getCategory();
     switch(c) {
     case PRIMITIVE:
-      return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
-          ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory());
+      return LazyPrimitiveObjectInspectorFactory.getLazyObjectInspector(
+          ((PrimitiveTypeInfo)typeInfo).getPrimitiveCategory(), escaped, escapeChar);
     case MAP:
-      return ObjectInspectorFactory.getLazySimpleMapObjectInspector(
-          createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapKeyTypeInfo(), separator, separatorIndex+2, nullSequence), 
-          createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapValueTypeInfo(), separator, separatorIndex+2, nullSequence), 
+      return LazyObjectInspectorFactory.getLazySimpleMapObjectInspector(
+          createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapKeyTypeInfo(), 
+              separator, separatorIndex+2, nullSequence, escaped, escapeChar), 
+          createLazyObjectInspector(((MapTypeInfo)typeInfo).getMapValueTypeInfo(), 
+              separator, separatorIndex+2, nullSequence, escaped, escapeChar), 
           separator[separatorIndex], 
           separator[separatorIndex+1], 
-          nullSequence);
+          nullSequence, escaped, escapeChar);
     case LIST: 
-      return ObjectInspectorFactory.getLazySimpleListObjectInspector(
-          createLazyObjectInspector(((ListTypeInfo)typeInfo).getListElementTypeInfo(), separator, separatorIndex+1, nullSequence),
+      return LazyObjectInspectorFactory.getLazySimpleListObjectInspector(
+          createLazyObjectInspector(((ListTypeInfo)typeInfo).getListElementTypeInfo(), 
+              separator, separatorIndex+1, nullSequence, escaped, escapeChar),
           separator[separatorIndex], 
-          nullSequence);
+          nullSequence, escaped, escapeChar);
     case STRUCT:
       StructTypeInfo structTypeInfo = (StructTypeInfo)typeInfo;
       List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
@@ -124,14 +151,15 @@
       List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());
       for(int i=0; i<fieldTypeInfos.size(); i++) {
         fieldObjectInspectors.add(
-            createLazyObjectInspector(fieldTypeInfos.get(i), separator, separatorIndex+1, nullSequence));
+            createLazyObjectInspector(fieldTypeInfos.get(i), separator, 
+                separatorIndex+1, nullSequence, escaped, escapeChar));
       }
-      return ObjectInspectorFactory.getLazySimpleStructObjectInspector(
+      return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
           fieldNames, 
           fieldObjectInspectors, 
           separator[separatorIndex], 
           nullSequence,
-          false);  
+          false, escaped, escapeChar);  
     }
 
     throw new RuntimeException("Hive LazySerDe Internal error.");
@@ -143,36 +171,36 @@
    * 
    * @param lastColumnTakesRest whether the last column of the struct should take
    *                            the rest of the row if there are extra fields. 
-   * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text)
+   * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte)
    */  
   public static ObjectInspector createLazyStructInspector(List<String> columnNames, 
       List<TypeInfo> typeInfos, byte[] separators, 
-      Text nullSequence, boolean lastColumnTakesRest) {
+      Text nullSequence, boolean lastColumnTakesRest, boolean escaped, byte escapeChar) {
     ArrayList<ObjectInspector> columnObjectInspectors =
         new ArrayList<ObjectInspector>(typeInfos.size());  
     for (int i=0; i<typeInfos.size(); i++) {
       columnObjectInspectors.add(
-          LazyFactory.createLazyObjectInspector(typeInfos.get(i), separators, 1, nullSequence));
+          LazyFactory.createLazyObjectInspector(typeInfos.get(i), separators, 1, nullSequence, escaped, escapeChar));
     }
     return 
-        ObjectInspectorFactory.getLazySimpleStructObjectInspector(columnNames,
-            columnObjectInspectors, separators[0], nullSequence, lastColumnTakesRest);
+        LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(columnNames,
+            columnObjectInspectors, separators[0], nullSequence, lastColumnTakesRest, escaped, escapeChar);
   }
   
   /**
    * Create a hierarchical ObjectInspector for ColumnarStruct with the given
    * columnNames and columnTypeInfos.
    * 
-   * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text)
+   * @see LazyFactory#createLazyObjectInspector(TypeInfo, byte[], int, Text, boolean, byte)
    */
   public static ObjectInspector createColumnarStructInspector(
       List<String> columnNames, List<TypeInfo> columnTypes, byte[] separators,
-      Text nullSequence) {
+      Text nullSequence, boolean escaped, byte escapeChar) {
     ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>(
         columnTypes.size());
     for (int i = 0; i < columnTypes.size(); i++) {
       columnObjectInspectors.add(LazyFactory.createLazyObjectInspector(
-          columnTypes.get(i), separators, 1, nullSequence));
+          columnTypes.get(i), separators, 1, nullSequence, escaped, escapeChar));
     }
     return ObjectInspectorFactory.getColumnarStructObjectInspector(columnNames,
         columnObjectInspectors, nullSequence);

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java Thu Jul  2 02:03:14 2009
@@ -19,6 +19,7 @@
 
 import java.nio.charset.CharacterCodingException;
 
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector;
 import org.apache.hadoop.io.FloatWritable;
 import org.apache.hadoop.io.Text;
 
@@ -26,12 +27,17 @@
  * LazyObject for storing a value of Double.
  * 
  */
-public class LazyFloat extends LazyPrimitive<FloatWritable> {
+public class LazyFloat extends LazyPrimitive<LazyFloatObjectInspector, FloatWritable> {
 
-  public LazyFloat() {
+  public LazyFloat(LazyFloatObjectInspector oi) {
+    super(oi);
     data = new FloatWritable();
   }
   
+  public LazyFloat(LazyFloat copy) {
+    super(copy);
+    data = new FloatWritable(copy.data.get());
+  }  
   
   @Override
   public void init(ByteArrayRef bytes, int start, int length) {

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java Thu Jul  2 02:03:14 2009
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyIntObjectInspector;
 import org.apache.hadoop.io.IntWritable;
 
 /**
@@ -35,12 +36,19 @@
  * </p>
  * 
  */
-public class LazyInteger extends LazyPrimitive<IntWritable> {
+public class LazyInteger extends LazyPrimitive<LazyIntObjectInspector, IntWritable> {
 
-  public LazyInteger() {
+  public LazyInteger(LazyIntObjectInspector oi) {
+    super(oi);
     data = new IntWritable();
   }
 
+  public LazyInteger(LazyInteger copy) {
+    super(copy);
+    data = new IntWritable(copy.data.get());
+  }  
+  
+  
   @Override
   public void init(ByteArrayRef bytes, int start, int length) {
     try {

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java Thu Jul  2 02:03:14 2009
@@ -20,6 +20,7 @@
 import java.io.IOException;
 import java.io.OutputStream;
 
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyLongObjectInspector;
 import org.apache.hadoop.io.LongWritable;
 
 /**
@@ -35,13 +36,19 @@
  * </p>
  * 
  */
-public class LazyLong extends LazyPrimitive<LongWritable> {
+public class LazyLong extends LazyPrimitive<LazyLongObjectInspector, LongWritable> {
 
 
-  public LazyLong() {
+  public LazyLong(LazyLongObjectInspector oi) {
+    super(oi);
     data = new LongWritable();
   }
 
+  public LazyLong(LazyLong copy) {
+    super(copy);
+    data = new LongWritable(copy.data.get());
+  }
+   
   @Override
   public void init(ByteArrayRef bytes, int start, int length) {
     try {

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyMap.java Thu Jul  2 02:03:14 2009
@@ -19,10 +19,12 @@
 
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.Map;
 
-import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.LazyMapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.io.Text;
 
 /**
@@ -30,9 +32,9 @@
  * Note that the keys of the map cannot contain null.
  * 
  * LazyMap does not deal with the case of a NULL map. That is handled
- * by LazyMapObjectInspector.
+ * by the parent LazyObject.
  */
-public class LazyMap extends LazyNonPrimitive {
+public class LazyMap extends LazyNonPrimitive<LazyMapObjectInspector> {
   
   /**
    * Whether the data is already parsed or not.
@@ -63,7 +65,7 @@
   /**
    * The keys are stored in an array of LazyPrimitives.
    */
-  LazyPrimitive<?>[] keyObjects;
+  LazyPrimitive<?,?>[] keyObjects;
   /**
    * Whether init() is called on keyObjects[i]. 
    */
@@ -80,11 +82,10 @@
   boolean[] valueInited;
   
   /**
-   * Construct a LazyMap object with the TypeInfo.
-   * @param typeInfo  the TypeInfo representing the type of this LazyMap.
+   * Construct a LazyMap object with the ObjectInspector.
    */
-  protected LazyMap(TypeInfo typeInfo) {
-    super(typeInfo);
+  protected LazyMap(LazyMapObjectInspector oi) {
+    super(oi);
   }
 
   /**
@@ -106,7 +107,7 @@
       int initialSize = 2;
       keyStart = new int[initialSize];
       keyEnd = new int[initialSize];
-      keyObjects = new LazyPrimitive<?>[initialSize];
+      keyObjects = new LazyPrimitive<?,?>[initialSize];
       valueObjects = new LazyObject[initialSize];
       keyInited = new boolean[initialSize];
       valueInited = new boolean[initialSize];
@@ -122,14 +123,15 @@
 
   /**
    * Parse the byte[] and fill keyStart, keyEnd.
-   * @param itemSeparator     The separator between different entries.
-   * @param keyValueSeparator The separator between key and value.
-   * @param nullSequence      The byte sequence representing NULL.
    */
-  private void parse(byte itemSeparator, byte keyValueSeparator, 
-      Text nullSequence) {
+  private void parse() {
     parsed = true;
     
+    byte itemSeparator = oi.getItemSeparator();
+    byte keyValueSeparator = oi.getKeyValueSeparator(); 
+    boolean isEscaped = oi.isEscaped();
+    byte escapeChar = oi.getEscapeChar();
+    
     // empty array?
     if (length == 0) {
       mapSize = 0;
@@ -167,7 +169,13 @@
           && bytes[elementByteEnd] == keyValueSeparator) {
         keyValueSeparatorPosition = elementByteEnd;
       }
-      elementByteEnd++;
+      if (isEscaped && bytes[elementByteEnd] == escapeChar
+          && elementByteEnd+1 < arrayByteEnd) {
+        // ignore the char after escape_char
+        elementByteEnd += 2;
+      } else {
+        elementByteEnd ++;
+      }
     }
     
     // This makes sure we can use the same formula to compute the
@@ -191,28 +199,23 @@
    * most cases, user only wants to get one or two values out of the map, and 
    * the cost of building up a HashMap is substantially higher.
    * 
-   * @param itemSeparator     The separator between different entries.
-   * @param keyValueSeparator The separator between key and value.
-   * @param nullSequence      The byte sequence representing NULL.
    * @param key               The key object that we are looking for.
    * @return The corresponding value object, or NULL if not found
    */
-  public Object getMapValueElement(byte itemSeparator, byte keyValueSeparator, 
-      Text nullSequence, Object key) {
+  public Object getMapValueElement(Object key) {
     if (!parsed) {
-      parse(itemSeparator, keyValueSeparator, nullSequence);
+      parse();
     }
-    
     // search for the key
     for (int i=0; i<mapSize; i++) {
-      LazyPrimitive<?> lazyKeyI = uncheckedGetKey(i, nullSequence);
+      LazyPrimitive<?,?> lazyKeyI = uncheckedGetKey(i);
       if (lazyKeyI == null) continue;
-      // getObject() will convert LazyPrimitive to actual primitive objects.
-      Object keyI = lazyKeyI.getObject();
+      // getWritableObject() will convert LazyPrimitive to actual primitive writable objects.
+      Object keyI = lazyKeyI.getWritableObject();
       if (keyI == null) continue;
       if (keyI.equals(key)) {
         // Got a match, return the value
-        LazyObject v = uncheckedGetValue(i, nullSequence);
+        LazyObject v = uncheckedGetValue(i);
         return v == null ? v : v.getObject();
       }
     }
@@ -225,7 +228,8 @@
    * @param index  The index into the array starting from 0
    * @param nullSequence  The byte sequence representing the NULL value
    */
-  private LazyObject uncheckedGetValue(int index, Text nullSequence) {
+  private LazyObject uncheckedGetValue(int index) {
+    Text nullSequence = oi.getNullSequence();
     int valueIBegin = keyEnd[index] + 1;
     int valueILength = keyStart[index+1] - 1 - valueIBegin;
     if (valueILength < 0 || 
@@ -238,7 +242,7 @@
       valueInited[index] = true;
       if (valueObjects[index] == null) {
         valueObjects[index] = LazyFactory.createLazyObject(
-            ((MapTypeInfo)typeInfo).getMapValueTypeInfo());
+            ((MapObjectInspector)oi).getMapValueObjectInspector());
       }
       valueObjects[index].init(bytes, valueIBegin, valueILength);
     }
@@ -250,7 +254,8 @@
    * @param index  The index into the array starting from 0
    * @param nullSequence  The byte sequence representing the NULL value
    */
-  private LazyPrimitive<?> uncheckedGetKey(int index, Text nullSequence) {
+  private LazyPrimitive<?,?> uncheckedGetKey(int index) {
+    Text nullSequence = oi.getNullSequence(); 
     int keyIBegin = keyStart[index];
     int keyILength = keyEnd[index] - keyStart[index];
     if (keyILength < 0 || 
@@ -264,7 +269,7 @@
       if (keyObjects[index] == null) {
         // Keys are always primitive
         keyObjects[index] = LazyFactory.createLazyPrimitiveClass(
-            ((MapTypeInfo)typeInfo).getMapKeyTypeInfo().getTypeName());
+            (PrimitiveObjectInspector)((MapObjectInspector)oi).getMapKeyObjectInspector());
       }
       keyObjects[index].init(bytes, keyIBegin, keyILength);
     }
@@ -276,34 +281,32 @@
    * But each LazyMap has a separate cachedMap so we won't overwrite the
    * data by accident.
    */
-  HashMap<Object, Object> cachedMap;
+  LinkedHashMap<Object, Object> cachedMap;
   
   /**
    * Return the map object representing this LazyMap.
-   * Note that the keyObjects will be Java primitive objects.
-   * @param itemSeparator     The separator between different entries.
-   * @param keyValueSeparator The separator between key and value.
-   * @param nullSequence      The byte sequence representing NULL.
+   * Note that the keyObjects will be Writable primitive objects.
    * @return the map object
    */
-  public Map<Object, Object> getMap(byte itemSeparator, byte keyValueSeparator,
-      Text nullSequence) {
+  public Map<Object, Object> getMap() {
     if (!parsed) {
-      parse(itemSeparator, keyValueSeparator, nullSequence);
+      parse();
     }
     if (cachedMap == null) {
-      cachedMap = new HashMap<Object, Object>();
+      // Use LinkedHashMap to provide deterministic order
+      cachedMap = new LinkedHashMap<Object, Object>();
+    } else {
+      cachedMap.clear();
     }
-    cachedMap.clear();
     
     // go through each element of the map
     for (int i = 0; i < mapSize; i++) {
-      LazyPrimitive<?> lazyKey = uncheckedGetKey(i, nullSequence);
+      LazyPrimitive<?,?> lazyKey = uncheckedGetKey(i);
       if (lazyKey == null) continue;
       Object key = lazyKey.getObject();
       // do not overwrite if there are duplicate keys
       if (key != null && !cachedMap.containsKey(key)) {
-        LazyObject lazyValue = uncheckedGetValue(i, nullSequence);
+        LazyObject lazyValue = uncheckedGetValue(i);
         Object value = (lazyValue == null ? null : lazyValue.getObject());
         cachedMap.put(key, value);
       }
@@ -313,15 +316,11 @@
 
   /**
    * Get the size of the map represented by this LazyMap.
-   * @param itemSeparator     The separator between different entries.
-   * @param keyValueSeparator The separator between key and value.
-   * @param nullSequence      The byte sequence representing NULL.
    * @return                  The size of the map, -1 for NULL map.
    */
-  public int getMapSize(byte itemSeparator, byte keyValueSeparator,
-      Text nullSequence) {
+  public int getMapSize() {
     if (!parsed) {
-      parse(itemSeparator, keyValueSeparator, nullSequence);
+      parse();
     }
     return mapSize;
   }

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyNonPrimitive.java Thu Jul  2 02:03:14 2009
@@ -17,25 +17,28 @@
  */
 package org.apache.hadoop.hive.serde2.lazy;
 
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 
 /**
  * LazyPrimitive stores a primitive Object in a LazyObject.
  */
-public abstract class LazyNonPrimitive implements LazyObject {
+public abstract class LazyNonPrimitive<OI extends ObjectInspector> extends LazyObject<OI> {
 
   protected ByteArrayRef bytes;
   protected int start;
   protected int length;
-  
+
   /**
-   * The TypeInfo for this LazyNonPrimitive. 
+   * Create a LazyNonPrimitive object with the specified ObjectInspector.
+   * @param oi  The ObjectInspector would have to have a hierarchy of 
+   *            LazyObjectInspectors with the leaf nodes being 
+   *            WritableObjectInspectors.  It's used both for accessing the
+   *            type hierarchy of the complex object, as well as getting
+   *            meta information (separator, nullSequence, etc) when parsing
+   *            the lazy object.
    */
-  TypeInfo typeInfo;
-  
-  protected LazyNonPrimitive(TypeInfo typeInfo) {
-    this.typeInfo = typeInfo;
+  protected LazyNonPrimitive(OI oi) {
+    super(oi);
     bytes = null;
     start = 0;
     length = 0;

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java Thu Jul  2 02:03:14 2009
@@ -17,14 +17,27 @@
  */
 package org.apache.hadoop.hive.serde2.lazy;
 
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
 /**
  * LazyObject stores an object in a range of bytes in a byte[].
  * 
  * A LazyObject can represent any primitive object or hierarchical object
  * like array, map or struct.
  */
-public interface LazyObject {
+public abstract class LazyObject<OI extends ObjectInspector> {
 
+  OI oi;
+  
+  /**
+   * Create a LazyObject.
+   * @param oi  Derived classes can access meta information about this Lazy
+   *            Object (e.g, separator, nullSequence, escaper) from it.
+   */
+  protected LazyObject(OI oi) {
+    this.oi = oi;
+  }
+  
   /**
    * Set the data for this LazyObject.
    * We take ByteArrayRef instead of byte[] so that we will be able to drop
@@ -35,12 +48,12 @@
    * @param length The length of the data, starting from "start"
    * @see ByteArrayRef
    */
-  void init(ByteArrayRef bytes, int start, int length);
+  public abstract void init(ByteArrayRef bytes, int start, int length);
 
   /**
    * If the LazyObject is a primitive Object, then deserialize it and return
    * the actual primitive Object.
    * Otherwise (array, map, struct), return this. 
    */
-  public Object getObject();
+  public abstract Object getObject();
 }

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java Thu Jul  2 02:03:14 2009
@@ -17,20 +17,40 @@
  */
 package org.apache.hadoop.hive.serde2.lazy;
 
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
 
 /**
  * LazyPrimitive stores a primitive Object in a LazyObject.
  */
-public abstract class LazyPrimitive<T> implements LazyObject {
+public abstract class LazyPrimitive<OI extends ObjectInspector, 
+    T extends Writable> extends LazyObject<OI> {
+
+  LazyPrimitive(OI oi) {
+    super(oi);
+  }
+
+  LazyPrimitive(LazyPrimitive<OI, T> copy) {
+    super(copy.oi);
+    isNull = copy.isNull;
+  }
 
   T data;
   boolean isNull = false;
 
   /**
-   * Returns the actual primitive object represented by this LazyObject.
+   * Returns the primitive object represented by this LazyObject.
+   * This is useful because it can make sure we have "null" for null objects.
    */
-  public T getObject() {
-    return isNull ? null : data;
+  public Object getObject() {
+    return isNull ? null : this;
   }
 
+  public T getWritableObject() {
+    return isNull ? null : data;
+  }
+  
+  public String toString() {
+    return isNull ? null : data.toString();
+  }
 }

Modified: hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java?rev=790435&r1=790434&r2=790435&view=diff
==============================================================================
--- hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java (original)
+++ hadoop/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java Thu Jul  2 02:03:14 2009
@@ -18,6 +18,7 @@
 package org.apache.hadoop.hive.serde2.lazy;
 
 import org.apache.hadoop.hive.serde2.io.ShortWritable;
+import org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyShortObjectInspector;
 
 /**
  * LazyObject for storing a value of Short.
@@ -32,12 +33,18 @@
  * </p>
  * 
  */
-public class LazyShort extends LazyPrimitive<ShortWritable> {
+public class LazyShort extends LazyPrimitive<LazyShortObjectInspector, ShortWritable> {
 
-  public LazyShort() {
+  public LazyShort(LazyShortObjectInspector oi) {
+    super(oi);
     data = new ShortWritable();
   }
   
+  public LazyShort(LazyShort copy) {
+    super(copy);
+    data = new ShortWritable(copy.data.get());
+  }
+  
   @Override
   public void init(ByteArrayRef bytes, int start, int length) {
     try {



Mime
View raw message