hive-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From heyongqi...@apache.org
Subject svn commit: r1150978 - in /hive/trunk: ql/src/java/org/apache/hadoop/hive/ql/parse/ serde/src/java/org/apache/hadoop/hive/serde2/columnar/ serde/src/java/org/apache/hadoop/hive/serde2/lazy/ serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/ serde...
Date Tue, 26 Jul 2011 03:22:13 GMT
Author: heyongqiang
Date: Tue Jul 26 03:22:09 2011
New Revision: 1150978

URL: http://svn.apache.org/viewvc?rev=1150978&view=rev
Log:
HIVE-956: add support of columnar binary serde (Krishna Kumar via He Yongqiang)

Added:
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDeBase.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/
    hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java
Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
    hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Tue Jul 26 03:22:09 2011
@@ -173,7 +173,9 @@ public abstract class BaseSemanticAnalyz
       case HiveParser.TOK_TBLRCFILE:
         inputFormat = RCFILE_INPUT;
         outputFormat = RCFILE_OUTPUT;
-        shared.serde = COLUMNAR_SERDE;
+        if (shared.serde == null) {
+          shared.serde = COLUMNAR_SERDE;
+        }
         storageFormat = true;
         break;
       case HiveParser.TOK_TABLEFILEFORMAT:

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java Tue Jul 26 03:22:09 2011
@@ -51,15 +51,7 @@ import org.apache.hadoop.io.Writable;
  * (2) ColumnarSerDe initialize ColumnarStruct's field directly. But under the
  * field level, it works like LazySimpleSerDe<br>
  */
-public class ColumnarSerDe implements SerDe {
-
-  // We need some initial values in case user don't call initialize()
-  private ObjectInspector cachedObjectInspector;
-
-  private long serializedSize;
-  private SerDeStats stats;
-  private boolean lastOperationSerialize;
-  private boolean lastOperationDeserialize;
+public class ColumnarSerDe extends ColumnarSerDeBase {
 
   @Override
   public String toString() {
@@ -104,65 +96,15 @@ public class ColumnarSerDe implements Se
         serdeParams.getNullSequence());
 
     int size = serdeParams.getColumnTypes().size();
-    field = new BytesRefWritable[size];
-    for (int i = 0; i < size; i++) {
-      field[i] = new BytesRefWritable();
-      serializeCache.set(i, field[i]);
-    }
-
+    super.initialize(size);
     LOG.debug("ColumnarSerDe initialized with: columnNames="
         + serdeParams.getColumnNames() + " columnTypes="
         + serdeParams.getColumnTypes() + " separator="
         + Arrays.asList(serdeParams.getSeparators()) + " nullstring="
         + serdeParams.getNullString());
-
-    serializedSize = 0;
-    stats = new SerDeStats();
-    lastOperationSerialize = false;
-    lastOperationDeserialize = false;
-  }
-
-  // The object for storing row data
-  ColumnarStruct cachedLazyStruct;
-
-  /**
-   * Deserialize a row from the Writable to a LazyObject.
-   */
-  public Object deserialize(Writable blob) throws SerDeException {
-
-    if (!(blob instanceof BytesRefArrayWritable)) {
-      throw new SerDeException(getClass().toString()
-          + ": expects BytesRefArrayWritable!");
-    }
-
-    BytesRefArrayWritable cols = (BytesRefArrayWritable) blob;
-    cachedLazyStruct.init(cols);
-    lastOperationSerialize = false;
-    lastOperationDeserialize = true;
-    return cachedLazyStruct;
-  }
-
-  /**
-   * Returns the ObjectInspector for the row.
-   */
-  public ObjectInspector getObjectInspector() throws SerDeException {
-    return cachedObjectInspector;
   }
 
   /**
-   * Returns the Writable Class after serialization.
-   *
-   * @see SerDe#getSerializedClass()
-   */
-  public Class<? extends Writable> getSerializedClass() {
-    return BytesRefArrayWritable.class;
-  }
-
-  BytesRefArrayWritable serializeCache = new BytesRefArrayWritable();
-  BytesRefWritable field[];
-  ByteStream.Output serializeStream = new ByteStream.Output();
-
-  /**
    * Serialize a row of data.
    *
    * @param obj
@@ -244,20 +186,4 @@ public class ColumnarSerDe implements Se
     return serializeCache;
   }
 
-  /**
-   * Returns the statistics after (de)serialization)
-   */
-
-  public SerDeStats getSerDeStats() {
-    // must be different
-    assert (lastOperationSerialize != lastOperationDeserialize);
-
-    if (lastOperationSerialize) {
-      stats.setRawDataSize(serializedSize);
-    } else {
-      stats.setRawDataSize(cachedLazyStruct.getRawDataSerializedSize());
-    }
-    return stats;
-
-  }
 }

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDeBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDeBase.java?rev=1150978&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDeBase.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDeBase.java Tue Jul 26 03:22:09 2011
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.columnar;
+
+import org.apache.hadoop.hive.serde2.ByteStream;
+import org.apache.hadoop.hive.serde2.SerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeStats;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Writable;
+
+public abstract class ColumnarSerDeBase implements SerDe {
+
+  // The object for storing row data
+  ColumnarStructBase cachedLazyStruct;
+  // We need some initial values in case user don't call initialize()
+  protected ObjectInspector cachedObjectInspector;
+
+  protected long serializedSize;
+  protected SerDeStats stats;
+  protected boolean lastOperationSerialize;
+  protected boolean lastOperationDeserialize;
+
+  BytesRefArrayWritable serializeCache = new BytesRefArrayWritable();
+  BytesRefWritable field[];
+  ByteStream.Output serializeStream = new ByteStream.Output();
+
+  @Override
+  public Object deserialize(Writable blob) throws SerDeException {
+    if (!(blob instanceof BytesRefArrayWritable)) {
+      throw new SerDeException(getClass().toString()
+            + ": expects BytesRefArrayWritable!");
+    }
+
+    BytesRefArrayWritable cols = (BytesRefArrayWritable) blob;
+    cachedLazyStruct.init(cols);
+    lastOperationSerialize = false;
+    lastOperationDeserialize = true;
+    return cachedLazyStruct;
+  }
+
+  @Override
+  public SerDeStats getSerDeStats() {
+    // must be different
+    assert (lastOperationSerialize != lastOperationDeserialize);
+
+    if (lastOperationSerialize) {
+      stats.setRawDataSize(serializedSize);
+    } else {
+      stats.setRawDataSize(cachedLazyStruct.getRawDataSerializedSize());
+    }
+    return stats;
+  }
+
+  @Override
+  public Class<? extends Writable> getSerializedClass() {
+    return BytesRefArrayWritable.class;
+  }
+
+  protected void initialize(int size) throws SerDeException {
+    field = new BytesRefWritable[size];
+    for (int i = 0; i < size; i++) {
+      field[i] = new BytesRefWritable();
+      serializeCache.set(i, field[i]);
+    }
+
+    serializedSize = 0;
+    stats = new SerDeStats();
+    lastOperationSerialize = false;
+    lastOperationDeserialize = false;
+  }
+
+  @Override
+  public ObjectInspector getObjectInspector() throws SerDeException {
+    return cachedObjectInspector;
+  }
+
+}

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStruct.java Tue Jul 26 03:22:09 2011
@@ -18,20 +18,15 @@
 
 package org.apache.hadoop.hive.serde2.columnar;
 
-import java.io.IOException;
 import java.util.ArrayList;
-import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.serde2.SerDeStatsStruct;
 import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
 import org.apache.hadoop.hive.serde2.lazy.LazyFactory;
-import org.apache.hadoop.hive.serde2.lazy.LazyObject;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
 import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
 import org.apache.hadoop.io.Text;
 
 /**
@@ -41,12 +36,10 @@ import org.apache.hadoop.io.Text;
  * lazy way.
  *
  */
-public class ColumnarStruct implements SerDeStatsStruct{
+public class ColumnarStruct extends ColumnarStructBase {
 
   private static final Log LOG = LogFactory.getLog(ColumnarStruct.class);
 
-  int[] prjColIDs = null; // list of projected column IDs
-
   Text nullSequence;
   int lengthNullSequence;
 
@@ -72,207 +65,28 @@ public class ColumnarStruct implements S
    */
   public ColumnarStruct(ObjectInspector oi,
       ArrayList<Integer> notSkippedColumnIDs, Text nullSequence) {
-    List<? extends StructField> fieldRefs = ((StructObjectInspector) oi)
-        .getAllStructFieldRefs();
-    int num = fieldRefs.size();
-
-    fieldInfoList = new FieldInfo[num];
-
+    super(oi, notSkippedColumnIDs);
     if (nullSequence != null) {
       this.nullSequence = nullSequence;
       this.lengthNullSequence = nullSequence.getLength();
     }
-
-    // if no columns is set to be skipped, add all columns in
-    // 'notSkippedColumnIDs'
-    if (notSkippedColumnIDs == null || notSkippedColumnIDs.size() == 0) {
-      for (int i = 0; i < num; i++) {
-        notSkippedColumnIDs.add(i);
-      }
-    }
-
-    for (int i = 0; i < num; i++) {
-      fieldInfoList[i] = new FieldInfo(
-          LazyFactory.createLazyObject(fieldRefs.get(i)
-          .getFieldObjectInspector()),
-          !notSkippedColumnIDs.contains(i));
-    }
-
-    // maintain a list of non-NULL column IDs
-    int min = notSkippedColumnIDs.size() > num ? num : notSkippedColumnIDs
-        .size();
-    prjColIDs = new int[min];
-    for (int i = 0, index = 0; i < notSkippedColumnIDs.size(); ++i) {
-      int readCol = notSkippedColumnIDs.get(i).intValue();
-      if (readCol < num) {
-        prjColIDs[index] = readCol;
-        index++;
-      }
-    }
-  }
-
-  /**
-   * Get one field out of the struct.
-   *
-   * If the field is a primitive field, return the actual object. Otherwise
-   * return the LazyObject. This is because PrimitiveObjectInspector does not
-   * have control over the object used by the user - the user simply directly
-   * use the Object instead of going through Object
-   * PrimitiveObjectInspector.get(Object).
-   *
-   * NOTE: separator and nullSequence has to be the same each time this method
-   * is called. These two parameters are used only once to parse each record.
-   *
-   * @param fieldID
-   *          The field ID
-   * @param nullSequence
-   *          The sequence for null value
-   * @return The field as a LazyObject
-   */
-  public Object getField(int fieldID) {
-    return fieldInfoList[fieldID].uncheckedGetField();
-  }
-
-  class FieldInfo {
-    LazyObject field;
-    /*
-     * use an array instead of only one object in case in future hive does not do
-     * the byte copy.
-     */
-    ByteArrayRef cachedByteArrayRef;
-    BytesRefWritable rawBytesField;
-    boolean inited;
-    boolean fieldSkipped;
-
-    public FieldInfo(LazyObject lazyObject, boolean fieldSkipped) {
-      field = lazyObject;
-      cachedByteArrayRef = new ByteArrayRef();
-      if (fieldSkipped) {
-        this.fieldSkipped = true;
-        inited = true;
-      } else {
-        inited = false;
-      }
-    }
-
-    /*
-     * ============================ [PERF] ===================================
-     * This function is called for every row. Setting up the selected/projected
-     * columns at the first call, and don't do that for the following calls.
-     * Ideally this should be done in the constructor where we don't need to
-     * branch in the function for each row.
-     * =========================================================================
-     */
-    public void init(BytesRefWritable col) {
-        if (col != null) {
-          rawBytesField= col;
-          inited = false;
-        } else {
-          // select columns that actually do not exist in the file.
-          fieldSkipped = true;
-        }
-    }
-
-    /**
-     * Return the uncompressed size of this field
-     */
-    public long getSerializedSize(){
-      if (rawBytesField == null) {
-        return 0;
-      }
-      return rawBytesField.getLength();
-    }
-
-    /**
-     * Get the field out of the row without checking parsed. This is called by
-     * both getField and getFieldsAsList.
-     *
-     * @param fieldID
-     *          The id of the field starting from 0.
-     * @param nullSequence
-     *          The sequence representing NULL value.
-     * @return The value of the field
-     */
-    protected Object uncheckedGetField() {
-      if (fieldSkipped) {
-        return null;
-      }
-      if (!inited) {
-        try {
-          cachedByteArrayRef.setData(rawBytesField.getData());
-        } catch (IOException e) {
-          throw new RuntimeException(e);
-        }
-        field.init(cachedByteArrayRef, rawBytesField
-            .getStart(), rawBytesField.getLength());
-        inited = true;
-      }
-
-
-      int fieldLen = rawBytesField.length;
-      if (fieldLen == lengthNullSequence) {
-        byte[] data = cachedByteArrayRef.getData();
-
-        if (LazyUtils.compare(data, rawBytesField.getStart(), fieldLen,
-            nullSequence.getBytes(), 0, lengthNullSequence) == 0) {
-          return null;
-        }
-      }
-
-      return field.getObject();
-
-    }
   }
 
-  FieldInfo[] fieldInfoList = null;
-
-
-  /*
-   * ============================ [PERF] ===================================
-   * This function is called for every row. Setting up the selected/projected
-   * columns at the first call, and don't do that for the following calls.
-   * Ideally this should be done in the constructor where we don't need to
-   * branch in the function for each row.
-   * =========================================================================
-   */
-  public void init(BytesRefArrayWritable cols) {
-    for (int i = 0; i < prjColIDs.length; ++i) {
-      int fieldIndex = prjColIDs[i];
-      if (fieldIndex < cols.size()) {
-        fieldInfoList[fieldIndex].init(cols.unCheckedGet(fieldIndex));
-      } else {
-        // select columns that actually do not exist in the file.
-        fieldInfoList[fieldIndex].init(null);
-      }
-    }
-  }
-
-  ArrayList<Object> cachedList;
-
-  /**
-   * Get the values of the fields as an ArrayList.
-   *
-   * @param nullSequence
-   *          The sequence for the NULL value
-   * @return The values of the fields as an ArrayList.
-   */
-  public ArrayList<Object> getFieldsAsList() {
-    if (cachedList == null) {
-      cachedList = new ArrayList<Object>();
-    } else {
-      cachedList.clear();
-    }
-    for (int i = 0; i < fieldInfoList.length; i++) {
-      cachedList.add(fieldInfoList[i].uncheckedGetField());
-    }
-    return cachedList;
-  }
-
-  public long getRawDataSerializedSize() {
-    long serializedSize = 0;
-    for (int i = 0; i < fieldInfoList.length; ++i) {
-      serializedSize += fieldInfoList[i].getSerializedSize();
-    }
-    return serializedSize;
+  @Override
+  protected int getLength(ObjectInspector objectInspector, ByteArrayRef cachedByteArrayRef,
+      int start, int fieldLen) {
+    if (fieldLen == lengthNullSequence) {
+      byte[] data = cachedByteArrayRef.getData();
+      if (LazyUtils.compare(data, start, fieldLen,
+          nullSequence.getBytes(), 0, lengthNullSequence) == 0) {
+        return -1;
+      }
+    }
+    return fieldLen;
+  }
+  
+  @Override
+  protected LazyObjectBase createLazyObjectBase(ObjectInspector objectInspector) {
+    return LazyFactory.createLazyObject(objectInspector);
   }
 }

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java?rev=1150978&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarStructBase.java Tue Jul 26 03:22:09 2011
@@ -0,0 +1,248 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.columnar;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.SerDeStatsStruct;
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+public abstract class ColumnarStructBase implements SerDeStatsStruct {
+
+  class FieldInfo {
+    LazyObjectBase field;
+    /*
+     * use an array instead of only one object in case in future hive does not do
+     * the byte copy.
+     */
+    ByteArrayRef cachedByteArrayRef;
+    BytesRefWritable rawBytesField;
+    boolean inited;
+    boolean fieldSkipped;
+    ObjectInspector objectInspector;
+
+    public FieldInfo(LazyObjectBase lazyObject, boolean fieldSkipped, ObjectInspector oi) {
+      field = lazyObject;
+      cachedByteArrayRef = new ByteArrayRef();
+      objectInspector = oi;
+      if (fieldSkipped) {
+        this.fieldSkipped = true;
+        inited = true;
+      } else {
+        inited = false;
+      }
+    }
+
+    /*
+     * ============================ [PERF] ===================================
+     * This function is called for every row. Setting up the selected/projected
+     * columns at the first call, and don't do that for the following calls.
+     * Ideally this should be done in the constructor where we don't need to
+     * branch in the function for each row.
+     * =========================================================================
+     */
+    public void init(BytesRefWritable col) {
+      if (col != null) {
+        rawBytesField = col;
+        inited = false;
+      } else {
+        // select columns that actually do not exist in the file.
+        fieldSkipped = true;
+      }
+    }
+
+    /**
+     * Return the uncompressed size of this field
+     */
+    public long getSerializedSize() {
+      if (rawBytesField == null) {
+        return 0;
+      }
+      return rawBytesField.getLength();
+    }
+
+    /**
+     * Get the field out of the row without checking parsed. This is called by
+     * both getField and getFieldsAsList.
+     *
+     * @return The value of the field
+     */
+    protected Object uncheckedGetField() {
+      if (fieldSkipped) {
+        return null;
+      }
+      if (!inited) {
+        try {
+          cachedByteArrayRef.setData(rawBytesField.getData());
+        } catch (IOException e) {
+          throw new RuntimeException(e);
+        }
+        inited = true;
+        int byteLength = getLength(objectInspector, cachedByteArrayRef, rawBytesField.getStart(),
+            rawBytesField.getLength());
+        if (byteLength == -1) {
+          return null;
+        }
+
+        field.init(cachedByteArrayRef, rawBytesField.getStart(), byteLength);
+        return field.getObject();
+      } else {
+        if (getLength(objectInspector, cachedByteArrayRef, rawBytesField.getStart(),
+            rawBytesField.getLength()) == -1) {
+          return null;
+        }
+        return field.getObject();
+      }
+    }
+  }
+
+  protected int[] prjColIDs = null;
+  private FieldInfo[] fieldInfoList = null;
+  private ArrayList<Object> cachedList;
+
+  public ColumnarStructBase(ObjectInspector oi,
+      ArrayList<Integer> notSkippedColumnIDs) {
+    List<? extends StructField> fieldRefs = ((StructObjectInspector) oi)
+        .getAllStructFieldRefs();
+    int num = fieldRefs.size();
+
+    fieldInfoList = new FieldInfo[num];
+
+    // if no columns is set to be skipped, add all columns in
+    // 'notSkippedColumnIDs'
+    if (notSkippedColumnIDs == null || notSkippedColumnIDs.size() == 0) {
+      for (int i = 0; i < num; i++) {
+        notSkippedColumnIDs.add(i);
+      }
+    }
+
+    for (int i = 0; i < num; i++) {
+      ObjectInspector foi = fieldRefs.get(i).getFieldObjectInspector();
+      fieldInfoList[i] = new FieldInfo(
+          createLazyObjectBase(foi),
+          !notSkippedColumnIDs.contains(i),
+          foi);
+    }
+
+    // maintain a list of non-NULL column IDs
+    int min = notSkippedColumnIDs.size() > num ? num : notSkippedColumnIDs
+        .size();
+    prjColIDs = new int[min];
+    for (int i = 0, index = 0; i < notSkippedColumnIDs.size(); ++i) {
+      int readCol = notSkippedColumnIDs.get(i).intValue();
+      if (readCol < num) {
+        prjColIDs[index] = readCol;
+        index++;
+      }
+    }
+  }
+
+  /**
+   * Get one field out of the struct.
+   * 
+   * If the field is a primitive field, return the actual object. Otherwise
+   * return the LazyObject. This is because PrimitiveObjectInspector does not
+   * have control over the object used by the user - the user simply directly
+   * use the Object instead of going through Object
+   * PrimitiveObjectInspector.get(Object).
+   * 
+   * NOTE: separator and nullSequence has to be the same each time this method
+   * is called. These two parameters are used only once to parse each record.
+   * 
+   * @param fieldID
+   *          The field ID
+   * @param nullSequence
+   *          The sequence for null value
+   * @return The field as a LazyObject
+   */
+  public Object getField(int fieldID) {
+    return fieldInfoList[fieldID].uncheckedGetField();
+  }
+
+  /**
+   * Check if the object is null and return the length of the stream
+   * 
+   * @param objectInspector
+   * @param cachedByteArrayRef
+   *          the bytes of the object
+   * @param start
+   *          the start offset
+   * @param length
+   *          the length
+   * 
+   * @return -1 for null, >=0 for length
+   */
+  protected abstract int getLength(ObjectInspector objectInspector,
+      ByteArrayRef cachedByteArrayRef, int start, int length);
+
+  /**
+   * create the lazy object for this field
+   * 
+   * @param objectInspector
+   *          the object inspector for the field
+   * @return the lazy object for the field
+   */
+  protected abstract LazyObjectBase createLazyObjectBase(ObjectInspector objectInspector);
+
+  public void init(BytesRefArrayWritable cols) {
+    for (int i = 0; i < prjColIDs.length; ++i) {
+      int fieldIndex = prjColIDs[i];
+      if (fieldIndex < cols.size()) {
+        fieldInfoList[fieldIndex].init(cols.unCheckedGet(fieldIndex));
+      } else {
+        // select columns that actually do not exist in the file.
+        fieldInfoList[fieldIndex].init(null);
+      }
+    }
+  }
+
+  /**
+   * Get the values of the fields as an ArrayList.
+   * 
+   * @param nullSequence
+   *          The sequence for the NULL value
+   * @return The values of the fields as an ArrayList.
+   */
+  public ArrayList<Object> getFieldsAsList() {
+    if (cachedList == null) {
+      cachedList = new ArrayList<Object>();
+    } else {
+      cachedList.clear();
+    }
+    for (int i = 0; i < fieldInfoList.length; i++) {
+      cachedList.add(fieldInfoList[i].uncheckedGetField());
+    }
+    return cachedList;
+  }
+
+  public long getRawDataSerializedSize() {
+    long serializedSize = 0;
+    for (int i = 0; i < fieldInfoList.length; ++i) {
+      serializedSize += fieldInfoList[i].getSerializedSize();
+    }
+    return serializedSize;
+  }
+
+}

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java?rev=1150978&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarSerDe.java Tue Jul 26 03:22:09 2011
@@ -0,0 +1,100 @@
+package org.apache.hadoop.hive.serde2.columnar;
+
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazyUtils;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.io.Writable;
+
+
+/**
+ * LazyBinaryColumnarSerDe. This serde combines elements of columnar serde and lazybinary serde
+ * to produce a serde which serializes columns into a BytesRefArrayWritable in a compact binary
+ * format and which is deserialized in a lazy, i.e. on-demand fashion.
+ * 
+ */
+public class LazyBinaryColumnarSerDe extends ColumnarSerDeBase {
+
+  private List<String> columnNames;
+  private List<TypeInfo> columnTypes;
+
+  @Override
+  public String toString() {
+    return getClass().toString()
+        + "["
+        + columnNames
+        + ":"
+        + columnTypes + "]";
+  }
+
+  @Override
+  public void initialize(Configuration conf, Properties tbl) throws SerDeException {
+    SerDeParameters serdeParams = new SerDeParameters();
+    LazyUtils.extractColumnInfo(tbl, serdeParams, getClass().getName());
+    columnNames = serdeParams.getColumnNames();
+    columnTypes = serdeParams.getColumnTypes();
+    
+    cachedObjectInspector = LazyBinaryFactory.createColumnarStructInspector(
+        columnNames, columnTypes);
+    java.util.ArrayList<Integer> notSkipIDs = ColumnProjectionUtils.getReadColumnIDs(conf);
+    cachedLazyStruct = new LazyBinaryColumnarStruct(cachedObjectInspector, notSkipIDs);
+    int size = columnTypes.size();
+    super.initialize(size);
+  }
+
+  static final byte[] INVALID_UTF__SINGLE_BYTE = {(byte)Integer.parseInt("10111111", 2)};
+  @Override
+  public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException {
+    if (objInspector.getCategory() != Category.STRUCT) {
+      throw new SerDeException(getClass().toString()
+          + " can only serialize struct types, but we got: "
+          + objInspector.getTypeName());
+    }
+
+    StructObjectInspector soi = (StructObjectInspector) objInspector;
+    List<? extends StructField> fields = soi.getAllStructFieldRefs();
+    List<Object> list = soi.getStructFieldsDataAsList(obj);
+
+    boolean warnedOnceNullMapKey = false;
+    serializeStream.reset();
+    serializedSize = 0;
+    int streamOffset = 0;
+    // Serialize each field
+    for (int i = 0; i < fields.size(); i++) {
+      // Get the field objectInspector and the field object.
+      ObjectInspector foi = fields.get(i).getFieldObjectInspector();
+      Object f = (list == null ? null : list.get(i));
+      //empty strings are marked by an invalid utf single byte sequence. A valid utf stream cannot 
+      //produce this sequence
+      if ((f != null) && (foi.getCategory().equals(ObjectInspector.Category.PRIMITIVE))
+          && ((PrimitiveObjectInspector) foi).getPrimitiveCategory().equals(
+              PrimitiveObjectInspector.PrimitiveCategory.STRING)
+          && ((StringObjectInspector) foi).getPrimitiveJavaObject(f).length() == 0) {
+        serializeStream.write(INVALID_UTF__SINGLE_BYTE, 0, 1);
+      } else {
+        LazyBinarySerDe.serialize(serializeStream, f, foi, true, warnedOnceNullMapKey);
+      }
+      field[i].set(serializeStream.getData(), streamOffset, serializeStream
+          .getCount()
+          - streamOffset);
+      streamOffset = serializeStream.getCount();
+    }
+    serializedSize = serializeStream.getCount();
+    lastOperationSerialize = true;
+    lastOperationDeserialize = false;
+    return serializeCache;
+  }
+}

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java?rev=1150978&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/columnar/LazyBinaryColumnarStruct.java Tue Jul 26 03:22:09 2011
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.columnar;
+
+import java.util.ArrayList;
+
+import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils.VInt;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+
+public class LazyBinaryColumnarStruct extends ColumnarStructBase {
+
+  public LazyBinaryColumnarStruct(ObjectInspector oi, ArrayList<Integer> notSkippedColumnIDs) {
+    super(oi, notSkippedColumnIDs);
+  }
+
+  static VInt vInt = new LazyBinaryUtils.VInt();
+
+  @Override
+  protected int getLength(ObjectInspector objectInspector, ByteArrayRef cachedByteArrayRef,
+      int start, int length) {
+    if (length == 0) {
+      return -1;
+    }
+    Category category = objectInspector.getCategory();
+    if (category.equals(Category.PRIMITIVE)) {
+      PrimitiveCategory primitiveCategory = ((PrimitiveObjectInspector) objectInspector)
+          .getPrimitiveCategory();
+      if (primitiveCategory.equals(PrimitiveCategory.STRING) && (length == 1) && 
+            (cachedByteArrayRef.getData()[start] 
+              == LazyBinaryColumnarSerDe.INVALID_UTF__SINGLE_BYTE[0])) {
+        return 0;
+      }
+    }
+    return length;
+  }
+
+  @Override
+  protected LazyObjectBase createLazyObjectBase(ObjectInspector objectInspector) {
+    return LazyBinaryFactory.createLazyBinaryObject(objectInspector);
+  }
+}

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFactory.java Tue Jul 26 03:22:09 2011
@@ -214,7 +214,7 @@ public final class LazyFactory {
           separators, 1, nullSequence, escaped, escapeChar));
     }
     return ObjectInspectorFactory.getColumnarStructObjectInspector(columnNames,
-        columnObjectInspectors, nullSequence);
+        columnObjectInspectors);
   }
 
   private LazyFactory() {

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObject.java Tue Jul 26 03:22:09 2011
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.serde2.obj
  * A LazyObject can represent any primitive object or hierarchical object like
  * array, map or struct.
  */
-public abstract class LazyObject<OI extends ObjectInspector> {
+public abstract class LazyObject<OI extends ObjectInspector> extends LazyObjectBase {
 
   OI oi;
 
@@ -40,27 +40,6 @@ public abstract class LazyObject<OI exte
     this.oi = oi;
   }
 
-  /**
-   * Set the data for this LazyObject. We take ByteArrayRef instead of byte[] so
-   * that we will be able to drop the reference to byte[] by a single
-   * assignment. The ByteArrayRef object can be reused across multiple rows.
-   * 
-   * @param bytes
-   *          The wrapper of the byte[].
-   * @param start
-   *          The start position inside the bytes.
-   * @param length
-   *          The length of the data, starting from "start"
-   * @see ByteArrayRef
-   */
-  public abstract void init(ByteArrayRef bytes, int start, int length);
-
-  /**
-   * If the LazyObject is a primitive Object, then deserialize it and return the
-   * actual primitive Object. Otherwise (array, map, struct), return this.
-   */
-  public abstract Object getObject();
-
   @Override
   public abstract int hashCode();
 

Added: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java?rev=1150978&view=auto
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java (added)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyObjectBase.java Tue Jul 26 03:22:09 2011
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.lazy;
+
+public abstract class LazyObjectBase {
+
+  /**
+   * Set the data for this LazyObjectBase. We take ByteArrayRef instead of byte[] so
+   * that we will be able to drop the reference to byte[] by a single
+   * assignment. The ByteArrayRef object can be reused across multiple rows.
+   * 
+   * @param bytes
+   *          The wrapper of the byte[].
+   * @param start
+   *          The start position inside the bytes.
+   * @param length
+   *          The length of the data, starting from "start"
+   * @see ByteArrayRef
+   */
+  public abstract void init(ByteArrayRef bytes, int start, int length);
+
+  /**
+   * If the LazyObjectBase is a primitive Object, then deserialize it and return the
+   * actual primitive Object. Otherwise (array, map, struct), return this.
+   */
+  public abstract Object getObject();
+
+}

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java Tue Jul 26 03:22:09 2011
@@ -19,7 +19,6 @@
 package org.apache.hadoop.hive.serde2.lazy;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
@@ -232,39 +231,7 @@ public class LazySimpleSerDe implements 
     serdeParams.lastColumnTakesRest = (lastColumnTakesRestString != null && lastColumnTakesRestString
         .equalsIgnoreCase("true"));
 
-    // Read the configuration parameters
-    String columnNameProperty = tbl.getProperty(Constants.LIST_COLUMNS);
-    // NOTE: if "columns.types" is missing, all columns will be of String type
-    String columnTypeProperty = tbl.getProperty(Constants.LIST_COLUMN_TYPES);
-
-    // Parse the configuration parameters
-
-    if (columnNameProperty != null && columnNameProperty.length() > 0) {
-      serdeParams.columnNames = Arrays.asList(columnNameProperty.split(","));
-    } else {
-      serdeParams.columnNames = new ArrayList<String>();
-    }
-    if (columnTypeProperty == null) {
-      // Default type: all string
-      StringBuilder sb = new StringBuilder();
-      for (int i = 0; i < serdeParams.columnNames.size(); i++) {
-        if (i > 0) {
-          sb.append(":");
-        }
-        sb.append(Constants.STRING_TYPE_NAME);
-      }
-      columnTypeProperty = sb.toString();
-    }
-
-    serdeParams.columnTypes = TypeInfoUtils
-        .getTypeInfosFromTypeString(columnTypeProperty);
-
-    if (serdeParams.columnNames.size() != serdeParams.columnTypes.size()) {
-      throw new SerDeException(serdeName + ": columns has "
-          + serdeParams.columnNames.size()
-          + " elements while columns.types has "
-          + serdeParams.columnTypes.size() + " elements!");
-    }
+    LazyUtils.extractColumnInfo(tbl, serdeParams, serdeName);
 
     // Create the LazyObject for storing the rows
     serdeParams.rowTypeInfo = TypeInfoFactory.getStructTypeInfo(

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java Tue Jul 26 03:22:09 2011
@@ -21,7 +21,13 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.charset.CharacterCodingException;
-
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Properties;
+
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe.SerDeParameters;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
@@ -31,6 +37,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
 import org.apache.hadoop.io.Text;
 
 /**
@@ -222,6 +229,43 @@ public final class LazyUtils {
     return hash;
   }
 
+  public static void extractColumnInfo(Properties tbl, SerDeParameters serdeParams,
+      String serdeName) throws SerDeException {
+    // Read the configuration parameters
+    String columnNameProperty = tbl.getProperty(Constants.LIST_COLUMNS);
+    // NOTE: if "columns.types" is missing, all columns will be of String type
+    String columnTypeProperty = tbl.getProperty(Constants.LIST_COLUMN_TYPES);
+
+    // Parse the configuration parameters
+
+    if (columnNameProperty != null && columnNameProperty.length() > 0) {
+      serdeParams.columnNames = Arrays.asList(columnNameProperty.split(","));
+    } else {
+      serdeParams.columnNames = new ArrayList<String>();
+    }
+    if (columnTypeProperty == null) {
+      // Default type: all string
+      StringBuilder sb = new StringBuilder();
+      for (int i = 0; i < serdeParams.columnNames.size(); i++) {
+        if (i > 0) {
+          sb.append(":");
+        }
+        sb.append(Constants.STRING_TYPE_NAME);
+      }
+      columnTypeProperty = sb.toString();
+    }
+
+    serdeParams.columnTypes = TypeInfoUtils
+        .getTypeInfosFromTypeString(columnTypeProperty);
+
+    if (serdeParams.columnNames.size() != serdeParams.columnTypes.size()) {
+      throw new SerDeException(serdeName + ": columns has "
+          + serdeParams.columnNames.size()
+          + " elements while columns.types has "
+          + serdeParams.columnTypes.size() + " elements!");
+    }
+  }
+
   private LazyUtils() {
     // prevent instantiation
   }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryFactory.java Tue Jul 26 03:22:09 2011
@@ -17,10 +17,14 @@
  */
 package org.apache.hadoop.hive.serde2.lazybinary;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryListObjectInspector;
 import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryMapObjectInspector;
 import org.apache.hadoop.hive.serde2.lazybinary.objectinspector.LazyBinaryStructObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableBooleanObjectInspector;
@@ -32,6 +36,7 @@ import org.apache.hadoop.hive.serde2.obj
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableShortObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableStringObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableVoidObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 
 /**
  * LazyBinaryFactory.
@@ -91,4 +96,16 @@ public final class LazyBinaryFactory {
   private LazyBinaryFactory() {
     // prevent instantiation
   }
+
+  public static ObjectInspector createColumnarStructInspector(List<String> columnNames,
+      List<TypeInfo> columnTypes) {
+    ArrayList<ObjectInspector> columnObjectInspectors = new ArrayList<ObjectInspector>(
+        columnTypes.size());
+    for (int i = 0; i < columnTypes.size(); i++) {
+      columnObjectInspectors
+          .add(LazyBinaryUtils.getLazyBinaryObjectInspectorFromTypeInfo(columnTypes.get(i)));
+    }
+    return ObjectInspectorFactory.getColumnarStructObjectInspector(columnNames,
+        columnObjectInspectors);
+  }
 }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinaryObject.java Tue Jul 26 03:22:09 2011
@@ -17,7 +17,7 @@
  */
 package org.apache.hadoop.hive.serde2.lazybinary;
 
-import org.apache.hadoop.hive.serde2.lazy.ByteArrayRef;
+import org.apache.hadoop.hive.serde2.lazy.LazyObjectBase;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 
 /**
@@ -27,7 +27,7 @@ import org.apache.hadoop.hive.serde2.obj
  * A LazyBinaryObject can represent any primitive object or hierarchical object
  * like string, list, map or struct.
  */
-public abstract class LazyBinaryObject<OI extends ObjectInspector> {
+public abstract class LazyBinaryObject<OI extends ObjectInspector> extends LazyObjectBase {
 
   OI oi;
 
@@ -42,30 +42,6 @@ public abstract class LazyBinaryObject<O
     this.oi = oi;
   }
 
-  /**
-   * Set the data for this LazyBinaryObject. We take ByteArrayRef instead of
-   * byte[] so that we will be able to drop the reference to byte[] by a single
-   * assignment. The ByteArrayRef object can be reused across multiple rows.
-   * 
-   * Never call this function if the object represent a null!!!
-   * 
-   * @param bytes
-   *          The wrapper of the byte[].
-   * @param start
-   *          The start position inside the bytes.
-   * @param length
-   *          The length of the data, starting from "start"
-   * @see ByteArrayRef
-   */
-  public abstract void init(ByteArrayRef bytes, int start, int length);
-
-  /**
-   * If the LazyBinaryObject is a primitive Object, then deserialize it and
-   * return the actual primitive Object. Otherwise (string, list, map, struct),
-   * return this.
-   */
-  public abstract Object getObject();
-
   @Override
   public abstract int hashCode();
 }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java Tue Jul 26 03:22:09 2011
@@ -179,6 +179,7 @@ public class LazyBinarySerDe implements 
    */
   BytesWritable serializeBytesWritable = new BytesWritable();
   ByteStream.Output serializeByteStream = new ByteStream.Output();
+  boolean nullMapKey = false;
 
   /**
    * Serialize an object to a byte buffer in a binary compact way.
@@ -195,8 +196,8 @@ public class LazyBinarySerDe implements 
 
     serializeByteStream.reset();
     // serialize the row as a struct
-    serializeStruct(serializeByteStream, obj,
-        (StructObjectInspector) objInspector);
+    nullMapKey = serializeStruct(serializeByteStream, obj,
+        (StructObjectInspector) objInspector, nullMapKey);
     // return the serialized bytes
     serializeBytesWritable.set(serializeByteStream.getData(), 0,
         serializeByteStream.getCount());
@@ -207,8 +208,6 @@ public class LazyBinarySerDe implements 
     return serializeBytesWritable;
   }
 
-  boolean nullMapKey = false;
-
   /**
    * Serialize a struct object without writing the byte size. This function is
    * shared by both row serialization and struct serialization.
@@ -219,12 +218,16 @@ public class LazyBinarySerDe implements 
    *          the struct object to serialize
    * @param objInspector
    *          the struct object inspector
+   * @param warnedOnceNullMapKey a boolean indicating whether a warning 
+   *          has been issued once already when encountering null map keys 
+   * @return a boolean indicating whether a warning for null map keys has been issued
+   *          once already
    */
-  private void serializeStruct(Output byteStream, Object obj,
-      StructObjectInspector soi) {
+  private static boolean serializeStruct(Output byteStream, Object obj,
+      StructObjectInspector soi, boolean warnedOnceNullMapKey) {
     // do nothing for null struct
     if (null == obj) {
-      return;
+      return warnedOnceNullMapKey;
     }
     /*
      * Interleave serializing one null byte and 8 struct fields in each round,
@@ -243,15 +246,16 @@ public class LazyBinarySerDe implements 
       // if this is the last element and serialize the
       // corresponding 8 struct fields at the same time
       if (7 == i % 8 || i == size - 1) {
-        serializeByteStream.write(nullByte);
+        byteStream.write(nullByte);
         for (int j = lasti; j <= i; j++) {
-          serialize(serializeByteStream, soi.getStructFieldData(obj, fields
-              .get(j)), fields.get(j).getFieldObjectInspector());
+          warnedOnceNullMapKey = serialize(byteStream, soi.getStructFieldData(obj, fields
+              .get(j)), fields.get(j).getFieldObjectInspector(), false, warnedOnceNullMapKey);
         }
         lasti = i + 1;
         nullByte = 0;
       }
     }
+    return warnedOnceNullMapKey;
   }
 
   /**
@@ -264,13 +268,19 @@ public class LazyBinarySerDe implements 
    *          the object to serialize
    * @param objInspector
    *          the object inspector
+   * @param skipLengthPrefix a boolean indicating whether length prefix is
+   *          needed for list/map/struct
+   * @param warnedOnceNullMapKey a boolean indicating whether a warning 
+   *          has been issued once already when encountering null map keys 
+   * @return a boolean indicating whether a warning for null map keys has been issued
+   *          once already
    */
-  private void serialize(Output byteStream, Object obj,
-      ObjectInspector objInspector) {
+  public static boolean serialize(Output byteStream, Object obj,
+      ObjectInspector objInspector, boolean skipLengthPrefix, boolean warnedOnceNullMapKey) {
 
     // do nothing for null object
     if (null == obj) {
-      return;
+      return warnedOnceNullMapKey;
     }
 
     switch (objInspector.getCategory()) {
@@ -278,37 +288,37 @@ public class LazyBinarySerDe implements 
       PrimitiveObjectInspector poi = (PrimitiveObjectInspector) objInspector;
       switch (poi.getPrimitiveCategory()) {
       case VOID: {
-        return;
+        return warnedOnceNullMapKey;
       }
       case BOOLEAN: {
         boolean v = ((BooleanObjectInspector) poi).get(obj);
         byteStream.write((byte) (v ? 1 : 0));
-        return;
+        return warnedOnceNullMapKey;
       }
       case BYTE: {
         ByteObjectInspector boi = (ByteObjectInspector) poi;
         byte v = boi.get(obj);
         byteStream.write(v);
-        return;
+        return warnedOnceNullMapKey;
       }
       case SHORT: {
         ShortObjectInspector spoi = (ShortObjectInspector) poi;
         short v = spoi.get(obj);
         byteStream.write((byte) (v >> 8));
         byteStream.write((byte) (v));
-        return;
+        return warnedOnceNullMapKey;
       }
       case INT: {
         IntObjectInspector ioi = (IntObjectInspector) poi;
         int v = ioi.get(obj);
         LazyBinaryUtils.writeVInt(byteStream, v);
-        return;
+        return warnedOnceNullMapKey;
       }
       case LONG: {
         LongObjectInspector loi = (LongObjectInspector) poi;
         long v = loi.get(obj);
         LazyBinaryUtils.writeVLong(byteStream, v);
-        return;
+        return warnedOnceNullMapKey;
       }
       case FLOAT: {
         FloatObjectInspector foi = (FloatObjectInspector) poi;
@@ -317,7 +327,7 @@ public class LazyBinarySerDe implements 
         byteStream.write((byte) (v >> 16));
         byteStream.write((byte) (v >> 8));
         byteStream.write((byte) (v));
-        return;
+        return warnedOnceNullMapKey;
       }
       case DOUBLE: {
         DoubleObjectInspector doi = (DoubleObjectInspector) poi;
@@ -330,18 +340,20 @@ public class LazyBinarySerDe implements 
         byteStream.write((byte) (v >> 16));
         byteStream.write((byte) (v >> 8));
         byteStream.write((byte) (v));
-        return;
+        return warnedOnceNullMapKey;
       }
       case STRING: {
         StringObjectInspector soi = (StringObjectInspector) poi;
         Text t = soi.getPrimitiveWritableObject(obj);
         /* write byte size of the string which is a vint */
         int length = t.getLength();
-        LazyBinaryUtils.writeVInt(byteStream, length);
+        if (!skipLengthPrefix) {
+          LazyBinaryUtils.writeVInt(byteStream, length);
+        }
         /* write string itself */
         byte[] data = t.getBytes();
         byteStream.write(data, 0, length);
-        return;
+        return warnedOnceNullMapKey;
       }
       default: {
         throw new RuntimeException("Unrecognized type: "
@@ -353,15 +365,18 @@ public class LazyBinarySerDe implements 
       ListObjectInspector loi = (ListObjectInspector) objInspector;
       ObjectInspector eoi = loi.getListElementObjectInspector();
 
-      // 1/ reserve spaces for the byte size of the list
-      // which is a integer and takes four bytes
-      int byteSizeStart = byteStream.getCount();
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      int listStart = byteStream.getCount();
-
+      int byteSizeStart = 0;
+      int listStart = 0;
+      if (!skipLengthPrefix) {
+        // 1/ reserve spaces for the byte size of the list
+        // which is a integer and takes four bytes
+        byteSizeStart = byteStream.getCount();
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        listStart = byteStream.getCount();
+      }
       // 2/ write the size of the list as a VInt
       int size = loi.getListLength(obj);
       LazyBinaryUtils.writeVInt(byteStream, size);
@@ -383,19 +398,21 @@ public class LazyBinarySerDe implements 
 
       // 4/ write element by element from the list
       for (int eid = 0; eid < size; eid++) {
-        serialize(byteStream, loi.getListElement(obj, eid), eoi);
+        warnedOnceNullMapKey = serialize(byteStream, loi.getListElement(obj, eid), eoi, 
+            false, warnedOnceNullMapKey);
       }
 
-      // 5/ update the list byte size
-      int listEnd = byteStream.getCount();
-      int listSize = listEnd - listStart;
-      byte[] bytes = byteStream.getData();
-      bytes[byteSizeStart] = (byte) (listSize >> 24);
-      bytes[byteSizeStart + 1] = (byte) (listSize >> 16);
-      bytes[byteSizeStart + 2] = (byte) (listSize >> 8);
-      bytes[byteSizeStart + 3] = (byte) (listSize);
-
-      return;
+      if (!skipLengthPrefix) {
+        // 5/ update the list byte size
+        int listEnd = byteStream.getCount();
+        int listSize = listEnd - listStart;
+        byte[] bytes = byteStream.getData();
+        bytes[byteSizeStart] = (byte) (listSize >> 24);
+        bytes[byteSizeStart + 1] = (byte) (listSize >> 16);
+        bytes[byteSizeStart + 2] = (byte) (listSize >> 8);
+        bytes[byteSizeStart + 3] = (byte) (listSize);
+      }
+      return warnedOnceNullMapKey;
     }
     case MAP: {
       MapObjectInspector moi = (MapObjectInspector) objInspector;
@@ -403,15 +420,19 @@ public class LazyBinarySerDe implements 
       ObjectInspector voi = moi.getMapValueObjectInspector();
       Map<?, ?> map = moi.getMap(obj);
 
-      // 1/ reserve spaces for the byte size of the map
-      // which is a integer and takes four bytes
-      int byteSizeStart = byteStream.getCount();
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      int mapStart = byteStream.getCount();
-
+      int byteSizeStart = 0;
+      int mapStart = 0;
+      if (!skipLengthPrefix) {
+        // 1/ reserve spaces for the byte size of the map
+        // which is a integer and takes four bytes
+        byteSizeStart = byteStream.getCount();
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        mapStart = byteStream.getCount();
+      }
+      
       // 2/ write the size of the map which is a VInt
       int size = map.size();
       LazyBinaryUtils.writeVInt(byteStream, size);
@@ -423,8 +444,8 @@ public class LazyBinarySerDe implements 
         // set the bit to 1 if a key is not null
         if (null != entry.getKey()) {
           nullByte |= 1 << (b % 8);
-        } else if (!nullMapKey) {
-          nullMapKey = true;
+        } else if (!warnedOnceNullMapKey) {
+          warnedOnceNullMapKey = true;
           LOG.warn("Null map key encountered! Ignoring similar problems.");
         }
         b++;
@@ -443,44 +464,50 @@ public class LazyBinarySerDe implements 
 
       // 4/ write key-value pairs one by one
       for (Map.Entry<?, ?> entry : map.entrySet()) {
-        serialize(byteStream, entry.getKey(), koi);
-        serialize(byteStream, entry.getValue(), voi);
+        warnedOnceNullMapKey = serialize(byteStream, entry.getKey(), koi, false, warnedOnceNullMapKey);
+        warnedOnceNullMapKey = serialize(byteStream, entry.getValue(), voi, false, warnedOnceNullMapKey);
       }
 
-      // 5/ update the byte size of the map
-      int mapEnd = byteStream.getCount();
-      int mapSize = mapEnd - mapStart;
-      byte[] bytes = byteStream.getData();
-      bytes[byteSizeStart] = (byte) (mapSize >> 24);
-      bytes[byteSizeStart + 1] = (byte) (mapSize >> 16);
-      bytes[byteSizeStart + 2] = (byte) (mapSize >> 8);
-      bytes[byteSizeStart + 3] = (byte) (mapSize);
-
-      return;
+      if (!skipLengthPrefix) {
+        // 5/ update the byte size of the map
+        int mapEnd = byteStream.getCount();
+        int mapSize = mapEnd - mapStart;
+        byte[] bytes = byteStream.getData();
+        bytes[byteSizeStart] = (byte) (mapSize >> 24);
+        bytes[byteSizeStart + 1] = (byte) (mapSize >> 16);
+        bytes[byteSizeStart + 2] = (byte) (mapSize >> 8);
+        bytes[byteSizeStart + 3] = (byte) (mapSize);
+      }
+      return warnedOnceNullMapKey;
     }
     case STRUCT: {
-      // 1/ reserve spaces for the byte size of the struct
-      // which is a integer and takes four bytes
-      int byteSizeStart = byteStream.getCount();
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      byteStream.write((byte) 0);
-      int structStart = byteStream.getCount();
-
+      int byteSizeStart = 0;
+      int structStart = 0;
+      if (!skipLengthPrefix) {
+        // 1/ reserve spaces for the byte size of the struct
+        // which is a integer and takes four bytes
+        byteSizeStart = byteStream.getCount();
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        byteStream.write((byte) 0);
+        structStart = byteStream.getCount();
+      }
       // 2/ serialize the struct
-      serializeStruct(byteStream, obj, (StructObjectInspector) objInspector);
+      warnedOnceNullMapKey = serializeStruct(byteStream, obj, (StructObjectInspector) objInspector,
+          warnedOnceNullMapKey);
 
-      // 3/ update the byte size of the struct
-      int structEnd = byteStream.getCount();
-      int structSize = structEnd - structStart;
-      byte[] bytes = byteStream.getData();
-      bytes[byteSizeStart] = (byte) (structSize >> 24);
-      bytes[byteSizeStart + 1] = (byte) (structSize >> 16);
-      bytes[byteSizeStart + 2] = (byte) (structSize >> 8);
-      bytes[byteSizeStart + 3] = (byte) (structSize);
-
-      return;
+      if (!skipLengthPrefix) {
+        // 3/ update the byte size of the struct
+        int structEnd = byteStream.getCount();
+        int structSize = structEnd - structStart;
+        byte[] bytes = byteStream.getData();
+        bytes[byteSizeStart] = (byte) (structSize >> 24);
+        bytes[byteSizeStart + 1] = (byte) (structSize >> 16);
+        bytes[byteSizeStart + 2] = (byte) (structSize >> 8);
+        bytes[byteSizeStart + 3] = (byte) (structSize);
+      }
+      return warnedOnceNullMapKey;
     }
     default: {
       throw new RuntimeException("Unrecognized type: "

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ColumnarStructObjectInspector.java Tue Jul 26 03:22:09 2011
@@ -23,12 +23,11 @@ import java.util.List;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.serde2.columnar.ColumnarStruct;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.hive.serde2.columnar.ColumnarStructBase;
 
 /**
  * ColumnarStructObjectInspector works on struct data that is stored in
- * ColumnarStruct.
+ * ColumnarStructBase.
  *
  * The names of the struct fields and the internal structure of the struct
  * fields are specified in the ctor of the ColumnarStructObjectInspector.
@@ -78,22 +77,18 @@ class ColumnarStructObjectInspector exte
     return ObjectInspectorUtils.getStandardStructTypeName(this);
   }
 
-  Text nullSequence;
-
   /**
    * Call ObjectInspectorFactory.getLazySimpleStructObjectInspector instead.
    */
   public ColumnarStructObjectInspector(List<String> structFieldNames,
-      List<ObjectInspector> structFieldObjectInspectors, Text nullSequence) {
-    init(structFieldNames, structFieldObjectInspectors, nullSequence);
+      List<ObjectInspector> structFieldObjectInspectors) {
+    init(structFieldNames, structFieldObjectInspectors);
   }
 
   protected void init(List<String> structFieldNames,
-      List<ObjectInspector> structFieldObjectInspectors, Text nullSequence) {
+      List<ObjectInspector> structFieldObjectInspectors) {
     assert (structFieldNames.size() == structFieldObjectInspectors.size());
 
-    this.nullSequence = nullSequence;
-
     fields = new ArrayList<MyField>(structFieldNames.size());
     for (int i = 0; i < structFieldNames.size(); i++) {
       fields.add(new MyField(i, structFieldNames.get(i),
@@ -101,14 +96,11 @@ class ColumnarStructObjectInspector exte
     }
   }
 
-  protected ColumnarStructObjectInspector(List<StructField> fields,
-      Text nullSequence) {
-    init(fields, nullSequence);
+  protected ColumnarStructObjectInspector(List<StructField> fields) {
+    init(fields);
   }
 
-  protected void init(List<StructField> fields, Text nullSequence) {
-    this.nullSequence = nullSequence;
-
+  protected void init(List<StructField> fields) {
     this.fields = new ArrayList<MyField>(fields.size());
     for (int i = 0; i < fields.size(); i++) {
       this.fields.add(new MyField(i, fields.get(i).getFieldName(), fields
@@ -138,7 +130,7 @@ class ColumnarStructObjectInspector exte
     if (data == null) {
       return null;
     }
-    ColumnarStruct struct = (ColumnarStruct) data;
+    ColumnarStructBase struct = (ColumnarStructBase) data;
     MyField f = (MyField) fieldRef;
 
     int fieldID = f.getFieldID();
@@ -152,7 +144,7 @@ class ColumnarStructObjectInspector exte
     if (data == null) {
       return null;
     }
-    ColumnarStruct struct = (ColumnarStruct) data;
+    ColumnarStructBase struct = (ColumnarStructBase) data;
     return struct.getFieldsAsList();
   }
 }

Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java?rev=1150978&r1=1150977&r2=1150978&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorFactory.java Tue Jul 26 03:22:09 2011
@@ -30,7 +30,6 @@ import java.util.Map;
 
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
 import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.io.Text;
 
 /**
  * ObjectInspectorFactory is the primary way to create new ObjectInspector
@@ -273,16 +272,15 @@ public final class ObjectInspectorFactor
 
   public static ColumnarStructObjectInspector getColumnarStructObjectInspector(
       List<String> structFieldNames,
-      List<ObjectInspector> structFieldObjectInspectors, Text nullSequence) {
+      List<ObjectInspector> structFieldObjectInspectors) {
     ArrayList<Object> signature = new ArrayList<Object>();
     signature.add(structFieldNames);
     signature.add(structFieldObjectInspectors);
-    signature.add(nullSequence.toString());
     ColumnarStructObjectInspector result = cachedColumnarStructObjectInspector
         .get(signature);
     if (result == null) {
       result = new ColumnarStructObjectInspector(structFieldNames,
-          structFieldObjectInspectors, nullSequence);
+          structFieldObjectInspectors);
       cachedColumnarStructObjectInspector.put(signature, result);
     }
     return result;

Added: hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java?rev=1150978&view=auto
==============================================================================
--- hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java (added)
+++ hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/columnar/TestLazyBinaryColumnarSerDe.java Tue Jul 26 03:22:09 2011
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.columnar;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TreeMap;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.CrossMapEqualComparer;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory.ObjectInspectorOptions;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.SimpleMapEqualComparer;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+public class TestLazyBinaryColumnarSerDe extends TestCase {
+
+  private static class InnerStruct {
+    public InnerStruct(Integer i, Long l) {
+      mInt = i;
+      mLong = l;
+    }
+    Integer mInt;
+    Long mLong;
+  }
+
+  private static class OuterStruct {
+    Byte mByte;
+    Short mShort;
+    Integer mInt;
+    Long mLong;
+    Float mFloat;
+    Double mDouble;
+    String mString;
+    List<InnerStruct> mArray;
+    Map<String, InnerStruct> mMap;
+    InnerStruct mStruct;
+  }
+
+  public void testSerDe() throws SerDeException {
+    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
+    String cols = ObjectInspectorUtils.getFieldNames(oi);
+    Properties props = new Properties();
+    props.setProperty(Constants.LIST_COLUMNS, cols);
+    props.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
+    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
+    serde.initialize(new Configuration(), props);
+
+    OuterStruct outerStruct = new OuterStruct();
+    outerStruct.mByte  = 1;
+    outerStruct.mShort = 2;
+    outerStruct.mInt = 3;
+    outerStruct.mLong = 4l;
+    outerStruct.mFloat = 5.01f;
+    outerStruct.mDouble = 6.001d;
+    outerStruct.mString = "seven";
+    InnerStruct is1 = new InnerStruct(8, 9l);
+    InnerStruct is2 = new InnerStruct(10, 11l);
+    outerStruct.mArray = new ArrayList<InnerStruct>(2);
+    outerStruct.mArray.add(is1);
+    outerStruct.mArray.add(is2);
+    outerStruct.mMap = new TreeMap<String, InnerStruct>();
+    outerStruct.mMap.put(new String("twelve"), new InnerStruct(13, 14l));
+    outerStruct.mMap.put(new String("fifteen"), new InnerStruct(16, 17l));
+    outerStruct.mStruct = new InnerStruct(18, 19l);
+    BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
+    
+    ObjectInspector out_oi = serde.getObjectInspector();
+    Object out_o = serde.deserialize(braw);
+    if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new CrossMapEqualComparer())) {
+      System.out.println("expected = "
+          + SerDeUtils.getJSONString(outerStruct, oi));
+      System.out.println("actual = "
+          + SerDeUtils.getJSONString(out_o, out_oi));
+      fail("Deserialized object does not compare");
+    }
+  }
+
+  public void testSerDeEmpties() throws SerDeException {
+    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
+    String cols = ObjectInspectorUtils.getFieldNames(oi);
+    Properties props = new Properties();
+    props.setProperty(Constants.LIST_COLUMNS, cols);
+    props.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
+    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
+    serde.initialize(new Configuration(), props);
+
+    OuterStruct outerStruct = new OuterStruct();
+    outerStruct.mByte  = 101;
+    outerStruct.mShort = 2002;
+    outerStruct.mInt = 3003;
+    outerStruct.mLong = 4004l;
+    outerStruct.mFloat = 5005.01f;
+    outerStruct.mDouble = 6006.001d;
+    outerStruct.mString = "";
+    outerStruct.mArray = new ArrayList<InnerStruct>();
+    outerStruct.mMap = new TreeMap<String, InnerStruct>();
+    outerStruct.mStruct = new InnerStruct(180018, 190019l);
+    BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
+    
+    ObjectInspector out_oi = serde.getObjectInspector();
+    Object out_o = serde.deserialize(braw);
+    if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new SimpleMapEqualComparer())) {
+      System.out.println("expected = "
+          + SerDeUtils.getJSONString(outerStruct, oi));
+      System.out.println("actual = "
+          + SerDeUtils.getJSONString(out_o, out_oi));
+      fail("Deserialized object does not compare");
+    }
+  }
+
+
+  public void testSerDeOuterNulls() throws SerDeException {
+    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
+    String cols = ObjectInspectorUtils.getFieldNames(oi);
+    Properties props = new Properties();
+    props.setProperty(Constants.LIST_COLUMNS, cols);
+    props.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
+    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
+    serde.initialize(new Configuration(), props);
+
+    OuterStruct outerStruct = new OuterStruct();
+    BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
+    
+    ObjectInspector out_oi = serde.getObjectInspector();
+    Object out_o = serde.deserialize(braw);
+    if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new SimpleMapEqualComparer())) {
+      System.out.println("expected = "
+          + SerDeUtils.getJSONString(outerStruct, oi));
+      System.out.println("actual = "
+          + SerDeUtils.getJSONString(out_o, out_oi));
+      fail("Deserialized object does not compare");
+    }
+  }
+  
+  public void testSerDeInnerNulls() throws SerDeException {
+    StructObjectInspector oi = (StructObjectInspector) ObjectInspectorFactory
+        .getReflectionObjectInspector(OuterStruct.class, ObjectInspectorOptions.JAVA);
+    String cols = ObjectInspectorUtils.getFieldNames(oi);
+    Properties props = new Properties();
+    props.setProperty(Constants.LIST_COLUMNS, cols);
+    props.setProperty(Constants.LIST_COLUMN_TYPES, ObjectInspectorUtils.getFieldTypes(oi));
+    LazyBinaryColumnarSerDe serde = new LazyBinaryColumnarSerDe();
+    serde.initialize(new Configuration(), props);
+
+    OuterStruct outerStruct = new OuterStruct();
+    outerStruct.mByte  = 1;
+    outerStruct.mShort = 2;
+    outerStruct.mInt = 3;
+    outerStruct.mLong = 4l;
+    outerStruct.mFloat = 5.01f;
+    outerStruct.mDouble = 6.001d;
+    outerStruct.mString = "seven";
+    InnerStruct is1 = new InnerStruct(null, 9l);
+    InnerStruct is2 = new InnerStruct(10, null);
+    outerStruct.mArray = new ArrayList<InnerStruct>(2);
+    outerStruct.mArray.add(is1);
+    outerStruct.mArray.add(is2);
+    outerStruct.mMap = new HashMap<String, InnerStruct>();
+    outerStruct.mMap.put(null, new InnerStruct(13, 14l));
+    outerStruct.mMap.put(new String("fifteen"), null);
+    outerStruct.mStruct = new InnerStruct(null, null);
+    BytesRefArrayWritable braw = (BytesRefArrayWritable) serde.serialize(outerStruct, oi);
+    
+    ObjectInspector out_oi = serde.getObjectInspector();
+    Object out_o = serde.deserialize(braw);
+    if (0 != ObjectInspectorUtils.compare(outerStruct, oi, out_o, out_oi, new SimpleMapEqualComparer())) {
+      System.out.println("expected = "
+          + SerDeUtils.getJSONString(outerStruct, oi));
+      System.out.println("actual = "
+          + SerDeUtils.getJSONString(out_o, out_oi));
+      fail("Deserialized object does not compare");
+    }
+  }
+
+
+}



Mime
View raw message