hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r706704 [22/23] - in /hadoop/core/trunk: ./ src/contrib/hive/ src/contrib/hive/bin/ src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/ src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/ src/contrib/hive/conf/ src/contrib/h...
Date Tue, 21 Oct 2008 18:11:18 GMT
Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java?rev=706704&r1=706703&r2=706704&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/MetadataTypedColumnsetSerDe.java Tue Oct 21 11:11:05 2008
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.hive.serde2;
 
-import java.io.UnsupportedEncodingException;
 import java.nio.charset.CharacterCodingException;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -44,33 +43,24 @@
 
   public static final Log LOG = LogFactory.getLog(MetadataTypedColumnsetSerDe.class.getName());
 
-  public String getShortName() {
-    return shortName();
-  }
-
-
-  public static String shortName() {
-    return "simple_meta";
-  }
-
   static {
     StackTraceElement[] sTrace = new Exception().getStackTrace();
     String className = sTrace[0].getClassName();
     try {
-      SerDeUtils.registerSerDe(shortName(), Class.forName(className));
-      // For backward compatibility: this class replaces the following class.
-      SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.simple_meta.MetadataTypedColumnsetSerDe", 
+      // For backward compatibility: this class replaces the columnsetSerDe class.
+      SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.thrift.columnsetSerDe", 
           Class.forName(className));
-    } catch(Exception e) {
+      } catch(Exception e) {
       throw new RuntimeException(e);
     }
   }
   
   final public static String DefaultSeparator = "\001";
-
   private String separator;
-  // constant for now, will make it configurable later.
-  private String nullString = "\\N"; 
+
+  final public static String defaultNullString = "\\N";
+  private String nullString; 
+
   private List<String> columnNames;
   private ObjectInspector cachedObjectInspector;
 
@@ -82,20 +72,36 @@
     separator = DefaultSeparator;
   }
 
-  public void initialize(Configuration job, Properties tbl) throws SerDeException {
-    separator = DefaultSeparator;
-    String alt_sep = tbl.getProperty(Constants.SERIALIZATION_FORMAT);
-    if(alt_sep != null && alt_sep.length() > 0) {
+  private String getByteValue(String altValue, String defaultVal) {
+    if (altValue != null && altValue.length() > 0) {
       try {
         byte b [] = new byte[1];
-        b[0] = Byte.valueOf(alt_sep).byteValue();
-        separator = new String(b);
+        b[0] = Byte.valueOf(altValue).byteValue();
+        return new String(b);
       } catch(NumberFormatException e) {
-        separator = alt_sep;
+        return altValue;
       }
     }
+    return defaultVal;
+  }
+
+  public void initialize(Configuration job, Properties tbl) throws SerDeException {
+    String alt_sep = tbl.getProperty(Constants.SERIALIZATION_FORMAT);
+    separator = getByteValue(alt_sep, DefaultSeparator);
+
+    String alt_null = tbl.getProperty(Constants.SERIALIZATION_NULL_FORMAT);
+    nullString = getByteValue(alt_null, defaultNullString);
+
     String columnProperty = tbl.getProperty("columns");
-    if (columnProperty == null || columnProperty.length() == 0) {
+    String serdeName = tbl.getProperty(Constants.SERIALIZATION_LIB);
+    // tables that were serialized with columnsetSerDe doesn't have metadata 
+    // so this hack applies to all such tables 
+    boolean columnsetSerDe = false;
+    if ((serdeName != null) && serdeName.equals("org.apache.hadoop.hive.serde.thrift.columnsetSerDe")) {
+      columnsetSerDe = true;
+    }
+    if (columnProperty == null || columnProperty.length() == 0 
+        || columnsetSerDe) {
       // Hack for tables with no columns
       // Treat it as a table with a single column called "col" 
       cachedObjectInspector = ObjectInspectorFactory.getReflectionObjectInspector(
@@ -104,9 +110,9 @@
       columnNames = Arrays.asList(columnProperty.split(","));
       cachedObjectInspector = MetadataListStructObjectInspector.getInstance(columnNames);
     }
-    LOG.info(getClass().getName() + ": initialized with columnNames: " + columnNames );
+    LOG.debug(getClass().getName() + ": initialized with columnNames: " + columnNames + " and separator code=" + (int)separator.charAt(0) );
   }
-
+  
   public static Object deserialize(ColumnSet c, String row, String sep, String nullString) throws Exception {
     if (c.col == null) {
       c.col = new ArrayList<String>();

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java?rev=706704&r1=706703&r2=706704&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/SerDeUtils.java Tue Oct 21 11:11:05 2008
@@ -170,7 +170,7 @@
     switch(oi.getCategory()) {
       case PRIMITIVE: {
         if (o == null) {
-          sb.append("\\N");
+          sb.append("null");
         } else if (o instanceof String) {
           sb.append(QUOTE);
           sb.append(escapeString((String)o));
@@ -184,54 +184,66 @@
         break;
       }
       case LIST: {
-        sb.append(LBRACKET);
         ListObjectInspector loi = (ListObjectInspector)oi;
         ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
         List<?> olist = loi.getList(o);
-        for (int i=0; i<olist.size(); i++) {
-          if (i>0) sb.append(COMMA);
-          buildJSONString(sb, olist.get(i), listElementObjectInspector);
+        if (olist == null) {
+          sb.append("null");
+        } else {
+          sb.append(LBRACKET);
+          for (int i=0; i<olist.size(); i++) {
+            if (i>0) sb.append(COMMA);
+            buildJSONString(sb, olist.get(i), listElementObjectInspector);
+          }
+          sb.append(RBRACKET);
         }
-        sb.append(RBRACKET);
         break;
       }
       case MAP: {
-        sb.append(LBRACE);
         MapObjectInspector moi = (MapObjectInspector)oi;
         ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
         ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
         Map<?,?> omap = moi.getMap(o);
-        boolean first = true;
-        for(Object entry : omap.entrySet()) {
-          if (first) {
-            first = false;
-          } else {
-            sb.append(COMMA);
+        if (omap == null) {
+          sb.append("null");
+        } else {
+          sb.append(LBRACE);
+          boolean first = true;
+          for(Object entry : omap.entrySet()) {
+            if (first) {
+              first = false;
+            } else {
+              sb.append(COMMA);
+            }
+            Map.Entry<?,?> e = (Map.Entry<?,?>)entry;
+            buildJSONString(sb, e.getKey(), mapKeyObjectInspector);
+            sb.append(COLON);
+            buildJSONString(sb, e.getValue(), mapValueObjectInspector);
           }
-          Map.Entry<?,?> e = (Map.Entry<?,?>)entry;
-          buildJSONString(sb, e.getKey(), mapKeyObjectInspector);
-          sb.append(COLON);
-          buildJSONString(sb, e.getValue(), mapValueObjectInspector);
+          sb.append(RBRACE);
         }
-        sb.append(RBRACE);
         break;
       }
       case STRUCT: {
-        sb.append(LBRACE);
         StructObjectInspector soi = (StructObjectInspector)oi;
         List<? extends StructField> structFields = soi.getAllStructFieldRefs();
-        for(int i=0; i<structFields.size(); i++) {
-          if (i>0) {
-            sb.append(COMMA);
+        if (structFields == null) {
+          sb.append("null");
+        } else {
+          sb.append(LBRACE);
+          for(int i=0; i<structFields.size(); i++) {
+            if (i>0) {
+              sb.append(COMMA);
+            }
+            sb.append(QUOTE);
+            sb.append(structFields.get(i).getFieldName());
+            sb.append(QUOTE);
+            sb.append(COLON);
+            buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), 
+                structFields.get(i).getFieldObjectInspector());          
           }
-          sb.append(QUOTE);
-          sb.append(structFields.get(i).getFieldName());
-          sb.append(QUOTE);
-          sb.append(COLON);
-          buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), 
-              structFields.get(i).getFieldObjectInspector());          
+          sb.append(RBRACE);
         }
-        sb.append(RBRACE);
         break;
       }
       default:

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java?rev=706704&r1=706703&r2=706704&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/Serializer.java Tue Oct 21 11:11:05 2008
@@ -54,5 +54,4 @@
    */
   public Writable serialize(Object obj, ObjectInspector objInspector) throws SerDeException;
 
-  public String getShortName();
 }

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftByteStreamTypedSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftByteStreamTypedSerDe.java?rev=706704&r1=706703&r2=706704&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftByteStreamTypedSerDe.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftByteStreamTypedSerDe.java Tue Oct 21 11:11:05 2008
@@ -22,9 +22,7 @@
 import java.util.Properties;
 
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
 import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
-import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.io.Writable;
 
 import com.facebook.thrift.TBase;
@@ -48,15 +46,6 @@
     throw new SerDeException("ThriftByteStreamTypedSerDe is still semi-abstract");
   }
 
-  public static String shortName() {
-    return "thriftbytestream";
-  }
-
-  public  String getShortName() {
-    return shortName();
-  }
-
-
   public ThriftByteStreamTypedSerDe(Type objectType, TProtocolFactory inFactory,
                                     TProtocolFactory outFactory) throws SerDeException {
     super(objectType);

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftDeserializer.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftDeserializer.java?rev=706704&r1=706703&r2=706704&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftDeserializer.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/ThriftDeserializer.java Tue Oct 21 11:11:05 2008
@@ -28,26 +28,6 @@
 
 public class ThriftDeserializer implements Deserializer {
 
-  public static String shortName() {
-    return "thrift";
-  }
-
-  public  String getShortName() {
-    return shortName();
-  }
-
-  static {
-    StackTraceElement[] sTrace = new Exception().getStackTrace();
-    String className = sTrace[0].getClassName();
-    try {
-      SerDeUtils.registerSerDe(shortName(), Class.forName(className));
-      // For backward compatibility: this class replaces the following class.
-      SerDeUtils.registerSerDe("org.apache.hadoop.hive.serde.thrift.ThriftSerDe", Class.forName(className));
-    } catch(Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-  
   private ThriftByteStreamTypedSerDe tsd;
 
   public ThriftDeserializer() { }

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDe.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,186 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
+
+import org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol;
+
+import java.util.*;
+import java.io.*;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.*;
+import org.apache.hadoop.util.StringUtils;
+
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.transport.*;
+
+public class DynamicSerDe implements SerDe, Serializable {
+
+  public static final Log LOG = LogFactory.getLog(DynamicSerDe.class.getName());
+
+  private String type_name;
+  private DynamicSerDeStructBase bt;
+
+  public static final String META_TABLE_NAME = "name";
+
+  transient private thrift_grammar parse_tree;
+  transient protected ByteStream.Input bis_;
+  transient protected ByteStream.Output bos_;
+
+  /**
+   * protocols are protected in case any of their properties need to be queried from another
+   * class in this package. For TCTLSeparatedProtocol for example, may want to query the separators.
+   */
+  transient protected TProtocol oprot_;
+  transient protected TProtocol iprot_;
+
+  TIOStreamTransport tios;
+
+  public void initialize(Configuration job, Properties tbl) throws SerDeException {
+    try {
+
+      String ddl = tbl.getProperty(Constants.SERIALIZATION_DDL);
+      type_name = tbl.getProperty(META_TABLE_NAME);
+      String protoName = tbl.getProperty(Constants.SERIALIZATION_FORMAT);
+
+      if(protoName == null) {
+        protoName = "com.facebook.thrift.protocol.TBinaryProtocol";
+      }
+      TProtocolFactory protFactory = TReflectionUtils.getProtocolFactoryByName(protoName);
+      bos_ = new ByteStream.Output();
+      bis_ = new ByteStream.Input();
+      tios = new TIOStreamTransport(bis_,bos_);
+
+      oprot_ = protFactory.getProtocol(tios);
+      iprot_ = protFactory.getProtocol(tios);
+
+      /**
+       * initialize the protocols
+       */
+
+      if(oprot_ instanceof org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol) {
+        ((ConfigurableTProtocol)oprot_).initialize(job, tbl);
+      }
+
+      if(iprot_ instanceof org.apache.hadoop.hive.serde2.thrift.ConfigurableTProtocol) {
+        ((ConfigurableTProtocol)iprot_).initialize(job, tbl);
+      }
+
+      // in theory the include path should come from the configuration
+      List<String> include_path = new ArrayList<String>();
+      include_path.add(".");
+      LOG.debug("ddl=" + ddl);
+      this.parse_tree = new thrift_grammar(new ByteArrayInputStream(ddl.getBytes()), include_path,false);
+      this.parse_tree.Start();
+
+      this.bt = (DynamicSerDeStructBase)this.parse_tree.types.get(type_name);
+
+      if(this.bt == null) {
+        this.bt = (DynamicSerDeStructBase)this.parse_tree.tables.get(type_name);
+      }
+
+      if(this.bt == null) {
+        throw new SerDeException("Could not lookup table type " + type_name + " in this ddl: " + ddl);
+      }
+
+      this.bt.initialize();
+    } catch (Exception e) {
+      System.out.println(StringUtils.stringifyException(e));
+      throw new SerDeException(e);
+    }
+  }
+
+  Object deserializeReuse = null;
+  public Object deserialize(Writable field) throws SerDeException {
+    try {
+      if (field instanceof Text) {
+        Text b = (Text)field;
+        bis_.reset(b.getBytes(), b.getLength());
+      } else {
+        BytesWritable b = (BytesWritable)field;
+        bis_.reset(b.get(), b.getSize());
+      }
+      deserializeReuse = this.bt.deserialize(deserializeReuse, iprot_);
+      return deserializeReuse;
+    } catch(Exception e) {
+      e.printStackTrace();
+      throw new SerDeException(e);
+    }
+  }
+
+  public static ObjectInspector dynamicSerDeStructBaseToObjectInspector(DynamicSerDeTypeBase bt) throws SerDeException {
+    if (bt.isList()) {
+      return ObjectInspectorFactory.getStandardListObjectInspector(
+          dynamicSerDeStructBaseToObjectInspector(((DynamicSerDeTypeList)bt).getElementType()));
+    } else if (bt.isMap()) {
+      DynamicSerDeTypeMap btMap = (DynamicSerDeTypeMap)bt; 
+      return ObjectInspectorFactory.getStandardMapObjectInspector(
+          dynamicSerDeStructBaseToObjectInspector(btMap.getKeyType()), 
+          dynamicSerDeStructBaseToObjectInspector(btMap.getValueType()));
+    } else if (bt.isPrimitive()) {
+      return ObjectInspectorFactory.getStandardPrimitiveObjectInspector(bt.getRealType());
+    } else {
+      // Must be a struct
+      DynamicSerDeStructBase btStruct = (DynamicSerDeStructBase)bt;
+      DynamicSerDeFieldList fieldList = btStruct.getFieldList();
+      DynamicSerDeField[] fields = fieldList.getChildren();
+      ArrayList<String> fieldNames = new ArrayList<String>(fields.length); 
+      ArrayList<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fields.length); 
+      for(int i=0; i<fields.length; i++) {
+        fieldNames.add(fields[i].name);
+        fieldObjectInspectors.add(
+            dynamicSerDeStructBaseToObjectInspector(fields[i].getFieldType().getMyType()));
+      }
+      return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
+    }
+  }
+
+  @Override
+  public ObjectInspector getObjectInspector() throws SerDeException {
+    return dynamicSerDeStructBaseToObjectInspector(this.bt);
+  }
+
+  @Override
+  public Class<? extends Writable> getSerializedClass() {
+    return BytesWritable.class;
+  }
+
+  BytesWritable ret = new BytesWritable();
+  @Override
+  public Writable serialize(Object obj, ObjectInspector objInspector)
+  throws SerDeException {
+    try {
+      bos_.reset();
+      this.bt.serialize(obj, objInspector, oprot_);
+      oprot_.getTransport().flush();
+    } catch(Exception e) {
+      e.printStackTrace();
+      throw new SerDeException(e);
+    }
+    ret.set(bos_.getData(),0,bos_.getCount());
+    return ret;
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeField.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import org.apache.hadoop.hive.serde2.*;
+import com.facebook.thrift.TException;
+import com.facebook.thrift.protocol.TProtocol;
+
+public class DynamicSerDeField extends DynamicSerDeSimpleNode {
+
+
+  // production is:
+  // [this.fieldid :] Requiredness() FieldType() this.name FieldValue() [CommaOrSemicolon()]
+
+  private final int FD_REQUIREDNESS = 0;
+  private final int FD_FIELD_TYPE = 1;
+  private final int FD_FIELD_VALUE =2;
+
+  public DynamicSerDeFieldType getFieldType() {
+    return (DynamicSerDeFieldType)this.jjtGetChild(FD_FIELD_TYPE);
+  }
+
+  public DynamicSerDeField(int i) {
+    super(i);
+  }
+  public DynamicSerDeField(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+}
+

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldList.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType.*;
+
+public class DynamicSerDeFieldList extends DynamicSerDeSimpleNode implements Serializable {
+
+  //  private void writeObject(ObjectOutputStream out) throws IOException {
+  //    out.writeObject(types_by_column_name);
+  //    out.writeObject(ordered_types);
+  //  }
+
+  // production: Field()*
+
+  // mapping of the fieldid to the field
+  private Map<Integer,DynamicSerDeTypeBase> types_by_id = null;
+  private Map<String,DynamicSerDeTypeBase> types_by_column_name = null;
+  private DynamicSerDeTypeBase ordered_types[] = null;
+
+  private Map<String,Integer> ordered_column_id_by_name = null;
+
+  public DynamicSerDeFieldList(int i) {
+    super(i);
+  }
+
+  public DynamicSerDeFieldList(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  private DynamicSerDeField getField(int i) {
+    return (DynamicSerDeField)this.jjtGetChild(i);
+  }
+
+  final public DynamicSerDeField [] getChildren() {
+    int size = this.jjtGetNumChildren();
+    DynamicSerDeField result [] = new DynamicSerDeField[size];
+    for(int i = 0; i < size; i++) {
+      result[i] = (DynamicSerDeField)this.jjtGetChild(i);
+    }
+    return result;
+  }
+
+  private int getNumFields() {
+    return this.jjtGetNumChildren();
+  }
+
+  public void initialize() {
+    if(types_by_id == null) {
+      // multiple means of lookup
+      types_by_id = new HashMap<Integer, DynamicSerDeTypeBase> ();
+      types_by_column_name = new HashMap<String, DynamicSerDeTypeBase> ();
+      ordered_types = new DynamicSerDeTypeBase[this.jjtGetNumChildren()];
+      ordered_column_id_by_name = new HashMap<String, Integer>();
+
+      // put them in and also roll them up while we're at it
+      // a Field contains a FieldType which in turn contains a type
+      for(int i = 0 ; i < this.jjtGetNumChildren(); i++) {
+        DynamicSerDeField mt = this.getField(i);
+        DynamicSerDeTypeBase type = mt.getFieldType().getMyType();
+        type.fieldid = mt.fieldid;
+        type.name = mt.name;
+
+        types_by_id.put(Integer.valueOf(mt.fieldid) , type);
+        types_by_column_name.put(mt.name, type);
+        ordered_types[i] = type;
+        ordered_column_id_by_name.put(mt.name, i);
+      }
+    }
+  }
+
+  private DynamicSerDeTypeBase getFieldByFieldId(int i) {
+    return types_by_id.get(i);
+  }
+
+  protected DynamicSerDeTypeBase getFieldByName(String fieldname) {
+    return types_by_column_name.get(fieldname);
+  }
+
+
+  public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    ArrayList<Object> struct = null;
+
+    if (reuse == null) {
+      struct = new ArrayList<Object>(this.getNumFields());
+      for(int i=0; i<this.getNumFields(); i++) {
+        struct.add(null);
+      }
+    } else {
+      struct = (ArrayList<Object>) reuse;
+      assert(struct.size() == this.getNumFields());
+    }
+
+    // Read the fields.
+    for(int i = 0; i < this.getNumFields(); i++) {
+      DynamicSerDeTypeBase mt = null;
+      TField field = null;
+
+      if (thrift_mode) {
+        field = iprot.readFieldBegin();
+
+        if(field.type >= 0) {
+          if(field.type == TType.STOP) {
+            break;
+          }
+          mt = this.getFieldByFieldId(field.id);
+          if(mt == null) {
+            System.err.println("ERROR for fieldid: " + field.id + " system has no knowledge of this field which is of type : " + field.type);
+            TProtocolUtil.skip(iprot,field.type);
+            continue;
+          }
+        }
+      }
+
+      // field.type < 0 means that this is a faked Thrift field, e.g., TControlSeparatedProtocol, which does not
+      // serialize the field id in the stream.  As a result, the only way to get the field id is to fall back to 
+      // the position "i".
+      // The intention of this hack (field.type < 0) is to make TControlSeparatedProtocol a real Thrift prototype,
+      // but there are a lot additional work to do to fulfill that, and that protocol inherently does not support
+      // versioning (adding/deleting fields).
+      int orderedId = -1;
+      if (!thrift_mode || field.type < 0) {
+        mt = this.ordered_types[i];
+        // We don't need to lookup order_column_id_by_name because we know it must be "i".
+        orderedId = i;
+      } else {
+        // Set the correct position
+        orderedId = ordered_column_id_by_name.get(mt.name); 
+      }
+      struct.set(orderedId, mt.deserialize(struct.get(orderedId), iprot));
+
+      if(thrift_mode) {
+        iprot.readFieldEnd();
+      }
+    }
+    return struct;
+  }
+
+
+  TField field = new TField();
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+
+    // Assuming the ObjectInspector represents exactly the same type as this struct.
+    // This assumption should be checked during query compile time.
+    assert(oi instanceof StructObjectInspector);
+    StructObjectInspector soi = (StructObjectInspector) oi;
+
+    // For every field
+    List<? extends StructField> fields = soi.getAllStructFieldRefs();
+    if (fields.size() != ordered_types.length) {
+      throw new SerDeException("Trying to serialize " + fields.size() 
+          + " fields into a struct with " + ordered_types.length);
+    }
+    for (int i=0; i<fields.size(); i++) {
+      Object f = soi.getStructFieldData(o, fields.get(i));
+      DynamicSerDeTypeBase mt = ordered_types[i];
+      
+      if(thrift_mode) {
+        field.name = mt.name;
+        field.type = mt.getType();
+        field.id = (short)mt.fieldid;
+        oprot.writeFieldBegin(field);
+      }
+
+      mt.serialize(f, fields.get(i).getFieldObjectInspector(), oprot);
+
+      if(thrift_mode) {
+        oprot.writeFieldEnd();
+      }
+    }
+    if(thrift_mode) {
+      oprot.writeFieldStop();
+    }
+  }
+
+  public String toString() {
+    StringBuffer result = new StringBuffer();
+    String prefix = "";
+    for(DynamicSerDeField t: this.getChildren()) {
+      result.append(prefix + t.fieldid + ":" + t.getFieldType().getMyType().toString() + " " + t.name);
+      prefix = ",";
+    }
+    return result.toString();
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFieldType.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+// basically just a container for the real type so more like a proxy
+public class DynamicSerDeFieldType extends DynamicSerDeSimpleNode {
+
+  // production: this.name | BaseType() | MapType() | SetType() | ListType()
+
+  private final int FD_FIELD_TYPE = 0;
+  public DynamicSerDeFieldType(int i) {
+    super(i);
+  }
+  public DynamicSerDeFieldType(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  protected DynamicSerDeTypeBase getMyType() {
+    // bugbug, need to deal with a named type here - i.e., look it up and proxy to it
+    // should raise an exception if this is a typedef since won't be any children
+    // and thus we can quickly find this comment and limitation.
+    return (DynamicSerDeTypeBase)this.jjtGetChild(FD_FIELD_TYPE);
+  }
+
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFunction.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFunction.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFunction.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeFunction.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.protocol.TMessageType;
+
+public class DynamicSerDeFunction extends DynamicSerDeStructBase {
+
+  // production is: Async() FunctionType() NAME FieldList() Throws() [CommaOrSemicolon]
+
+  private final int FD_ASYNC = 0;
+  private final int FD_FUNCTION_TYPE = 1;
+  private final int FD_FIELD_LIST = 2;
+  private final int FD_THROWS = 3;
+
+  public DynamicSerDeFunction(int i) {
+    super(i);
+  }
+  public DynamicSerDeFunction(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public DynamicSerDeFieldList getFieldList() {
+    return (DynamicSerDeFieldList)this.jjtGetChild(FD_FIELD_LIST);
+  }
+
+  public String toString() {
+    String result = "function " + this.name + " (";
+    result += this.getFieldList().toString();
+    result += ")";
+    return result;
+  }
+
+  public byte getType() {
+    return TMessageType.CALL;
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSimpleNode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSimpleNode.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSimpleNode.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeSimpleNode.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeSimpleNode extends SimpleNode {
+  protected static final boolean thrift_mode = true;
+  public DynamicSerDeSimpleNode(int i) {
+    super(i);
+  }
+  public DynamicSerDeSimpleNode(thrift_grammar p, int i) {
+    super(p,i);
+  }
+  protected int fieldid;
+  protected String name;
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStruct.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStruct.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStruct.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStruct.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeStruct extends DynamicSerDeStructBase {
+
+  // production is: struct this.name { FieldList() }
+
+  final private static int FD_FIELD_LIST = 0;
+
+  public DynamicSerDeStruct(int i) {
+    super(i);
+  }
+  public DynamicSerDeStruct(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public String toString() {
+    String result = "struct " + this.name + "(";
+    result += this.getFieldList().toString();
+    result += ")";
+    return result;
+  }
+
+  public DynamicSerDeFieldList getFieldList() {
+    return (DynamicSerDeFieldList)this.jjtGetChild(FD_FIELD_LIST);
+  }
+
+  public byte getType() {
+    return TType.STRUCT;
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeStructBase.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.protocol.*;
+import java.io.*;
+import java.util.List;
+
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+abstract public class DynamicSerDeStructBase extends DynamicSerDeTypeBase implements Serializable {
+
+  DynamicSerDeFieldList fieldList;
+
+  public DynamicSerDeStructBase(int i) {
+    super(i);
+  }
+  public DynamicSerDeStructBase(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  abstract public DynamicSerDeFieldList getFieldList();
+
+  public void initialize() {
+    fieldList = getFieldList();
+    fieldList.initialize();
+  }
+
+  public boolean isPrimitive() { return false; }
+
+
+  public Class getRealType() {
+    return List.class;
+  }
+
+  public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    if(thrift_mode) {
+      iprot.readStructBegin();
+    }
+
+    Object o = fieldList.deserialize(reuse, iprot);
+
+    if(thrift_mode) {
+      iprot.readStructEnd();
+    }
+    return o;
+  }
+
+  /**
+   * serialize
+   *
+   * The way to serialize a Thrift "table" which in thrift land is really a function and thus this class's name.
+   *
+   * @param o - this list should be in the order of the function's params for now. If we wanted to remove this requirement,
+   * we'd need to make it a List<Pair<String, Object>> with the String being the field name.
+   *
+   */
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    if(thrift_mode) {
+      oprot.writeStructBegin(new TStruct(this.name));
+    }
+
+    fieldList.serialize(o, oi, oprot);
+
+    if(thrift_mode) {
+      oprot.writeFieldStop();
+      oprot.writeStructEnd();
+    }
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+import com.facebook.thrift.protocol.TProtocol;
+import com.facebook.thrift.TException;
+import java.io.Serializable;
+
+public abstract class DynamicSerDeTypeBase extends DynamicSerDeSimpleNode implements Serializable {
+  private static final long serialVersionUID = 1L;
+
+  public DynamicSerDeTypeBase(int i) {
+    super(i);
+  }
+
+  public DynamicSerDeTypeBase(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public Class getRealType() throws SerDeException {
+    throw new SerDeException("Not implemented in base");
+  }
+
+  public Object get(Object obj) {
+    throw new RuntimeException("Not implemented in base");
+  }
+
+  public abstract Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException;
+
+  public abstract void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException;
+
+  public String toString() {
+    return "BAD";
+  }
+
+  public byte getType() {
+    return -1;
+  }
+
+  public boolean isPrimitive() { return true; }
+  public boolean isList() { return false; }
+  public boolean isMap() { return false; }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.protocol.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypeBool extends DynamicSerDeTypeBase {
+
+  // production is: bool
+
+  public DynamicSerDeTypeBool(int i) {
+    super(i);
+  }
+
+  public DynamicSerDeTypeBool(thrift_grammar p, int i) {
+    super(p, i);
+  }
+
+  public String toString() {
+    return "bool";
+  }
+
+  @Override
+  public Object deserialize(Object reuse, TProtocol iprot)
+      throws SerDeException, TException, IllegalAccessException {
+    return Boolean.valueOf(iprot.readBool());
+  }
+
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot)
+      throws TException, SerDeException, NoSuchFieldException,
+      IllegalAccessException {
+    assert (oi.getCategory() == ObjectInspector.Category.PRIMITIVE);
+    assert (((PrimitiveObjectInspector) oi).getPrimitiveClass()
+        .equals(Boolean.class));
+    oprot.writeBool((Boolean) o);
+  }
+
+  public byte getType() {
+    return TType.BOOL;
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.protocol.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypeByte extends DynamicSerDeTypeBase {
+
+  // production is: byte
+
+
+  public DynamicSerDeTypeByte(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypeByte(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public String toString() { return "byte"; }
+
+  public Byte deserialize(TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    return Byte.valueOf(iprot.readByte());
+  }
+  public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    return Byte.valueOf(iprot.readByte());
+  }
+
+  public void serialize(Object s, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    // bugbug need to use object of byte type!!!
+    oprot.writeByte((Byte)s);
+  }
+
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    assert(oi.getCategory() == ObjectInspector.Category.PRIMITIVE);
+    assert(((PrimitiveObjectInspector)oi).getPrimitiveClass().equals(Byte.class));
+    oprot.writeByte((Byte)o);
+  }
+
+  public byte getType() {
+    return TType.BYTE;
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypeDouble extends DynamicSerDeTypeBase {
+
+  // production is: double
+
+  public DynamicSerDeTypeDouble(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypeDouble(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public String toString() { return "double"; }
+
+  public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    return Double.valueOf(iprot.readDouble());
+  }
+
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    assert(oi.getCategory() == ObjectInspector.Category.PRIMITIVE);
+    assert(((PrimitiveObjectInspector)oi).getPrimitiveClass().equals(Double.class));
+    oprot.writeDouble((Double)o);
+  }
+
+  public byte getType() {
+    return TType.DOUBLE;
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.protocol.*;
+import java.util.*;
+import org.apache.hadoop.hive.serde.Constants;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypeList extends DynamicSerDeTypeBase {
+
+  public boolean isPrimitive() { return false; }
+  public boolean isList() { return true; }
+
+  // production is: list<FieldType()>
+
+  static final private int FD_TYPE = 0;
+
+  public Class getRealType() {
+    return java.util.ArrayList.class;
+  }
+
+  public DynamicSerDeTypeList(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypeList(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public DynamicSerDeTypeBase getElementType() {
+    return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_TYPE)).getMyType();
+  }
+
+  public String toString() {
+    return Constants.LIST_TYPE_NAME + "<" + this.getElementType().toString()  + ">";
+  }
+
+  @Override
+  public ArrayList<Object> deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    TList thelist = iprot.readListBegin();
+    ArrayList<Object> deserializeReuse;
+    if (reuse != null) {
+      deserializeReuse = (ArrayList<Object>)reuse;
+      // Trim to the size needed
+      while (deserializeReuse.size() > thelist.size) {
+        deserializeReuse.remove(deserializeReuse.size()-1);
+      }
+    } else {
+      deserializeReuse = new ArrayList<Object>();
+    }
+    deserializeReuse.ensureCapacity(thelist.size);
+    for(int i = 0; i < thelist.size; i++) {
+      if (i+1 > deserializeReuse.size()) {
+        deserializeReuse.add(this.getElementType().deserialize(null, iprot));
+      } else {
+        deserializeReuse.set(i, 
+            this.getElementType().deserialize(deserializeReuse.get(i), iprot));
+      }
+    }
+    // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy
+    iprot.readListEnd();
+    return deserializeReuse;
+  }
+
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    ListObjectInspector loi = (ListObjectInspector)oi;
+    ObjectInspector elementObjectInspector = loi.getListElementObjectInspector();
+    DynamicSerDeTypeBase mt = this.getElementType();
+
+    if (o instanceof List) {
+      List<?> list = (List<?>)o;
+      oprot.writeListBegin(new TList(mt.getType(),list.size()));
+      for (Object element: list) {
+        mt.serialize(element, elementObjectInspector, oprot);
+      }
+    } else {
+      Object[] list = (Object[])o;
+      oprot.writeListBegin(new TList(mt.getType(),list.length));
+      for (Object element: list) {
+        mt.serialize(element, elementObjectInspector, oprot);
+      }
+    }
+    // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy
+    oprot.writeListEnd();
+  }
+
+  public byte getType() {
+    return TType.LIST;
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypeMap extends DynamicSerDeTypeBase {
+
+  public boolean isPrimitive() { return false; }
+  public boolean isMap() { return true;}
+
+  // production is: Map<FieldType(),FieldType()>
+
+  private final byte FD_KEYTYPE = 0;
+  private final byte FD_VALUETYPE = 1;
+
+  // returns Map<?,?>
+  public Class getRealType() {
+    try {
+      Class c = this.getKeyType().getRealType();
+      Class c2 = this.getValueType().getRealType();
+      Object o = c.newInstance();
+      Object o2 = c2.newInstance();
+      Map<?,?> l = Collections.singletonMap(o,o2);
+      return l.getClass();
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw new RuntimeException(e);
+    }
+  }
+
+  public DynamicSerDeTypeMap(int i) {
+    super(i);
+  }
+
+  public DynamicSerDeTypeMap(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public DynamicSerDeTypeBase getKeyType() {
+    return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_KEYTYPE)).getMyType();
+  }
+
+  public DynamicSerDeTypeBase getValueType() {
+    return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_VALUETYPE)).getMyType();
+  }
+
+  public String toString() {
+    return "map<" + this.getKeyType().toString() + "," + this.getValueType().toString() + ">";
+  }
+
+  public Map<Object,Object> deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    HashMap<Object, Object> deserializeReuse;
+    if (reuse != null) {
+      deserializeReuse = (HashMap<Object, Object>)reuse;
+      deserializeReuse.clear();
+    } else {
+      deserializeReuse = new HashMap<Object, Object>();
+    }
+    TMap themap = iprot.readMapBegin();
+    for(int i = 0; i < themap.size; i++) {
+      Object key = this.getKeyType().deserialize(null, iprot);
+      Object value = this.getValueType().deserialize(null, iprot);
+      deserializeReuse.put(key,value);
+    }
+
+    // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy
+    iprot.readMapEnd();
+    return deserializeReuse;
+  }
+
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot)
+  throws TException, SerDeException, NoSuchFieldException,
+  IllegalAccessException {
+    DynamicSerDeTypeBase keyType = this.getKeyType();
+    DynamicSerDeTypeBase valueType = this.getValueType();
+
+    assert(oi.getCategory() == ObjectInspector.Category.MAP);
+    MapObjectInspector moi = (MapObjectInspector)oi;
+    ObjectInspector koi = moi.getMapKeyObjectInspector();
+    ObjectInspector voi = moi.getMapValueObjectInspector();
+
+    Map<?,?> map = moi.getMap(o);
+    oprot.writeMapBegin(new TMap(keyType.getType(),valueType.getType(),map.size()));
+    for(Iterator i = map.entrySet().iterator(); i.hasNext(); ) {
+      Map.Entry it = (Map.Entry)i.next();
+      Object key = it.getKey();
+      Object value = it.getValue();
+      keyType.serialize(key, koi, oprot);
+      valueType.serialize(value, voi, oprot);
+    }
+    // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy
+    oprot.writeMapEnd();      
+  }
+
+  public byte getType() {
+    return TType.MAP;
+  }
+};
+

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypeSet extends DynamicSerDeTypeBase {
+
+  // production is: set<FieldType()>
+
+  static final private int FD_TYPE = 0;
+
+  public DynamicSerDeTypeSet(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypeSet(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  // returns Set<?>
+  public Class getRealType() {
+    try {
+      Class c = this.getElementType().getRealType();
+      Object o = c.newInstance();
+      Set<?> l = Collections.singleton(o);
+      return l.getClass();
+    } catch (Exception e) {
+      e.printStackTrace();
+      throw new RuntimeException(e);
+    }
+  }
+
+  public DynamicSerDeTypeBase getElementType() {
+    return (DynamicSerDeTypeBase)((DynamicSerDeFieldType)this.jjtGetChild(FD_TYPE)).getMyType();
+  }
+
+  public String toString() {
+    return "set<" + this.getElementType().toString()  + ">";
+  }
+
+  public byte getType() {
+    return TType.SET;
+  }
+
+  /** NOTE: Set is not supported by Hive yet.
+   */
+  @Override
+  public Object deserialize(Object reuse, TProtocol iprot)
+  throws SerDeException, TException, IllegalAccessException {
+    TSet theset = iprot.readSetBegin();
+    Set<Object> result;
+    if (reuse != null) {
+      result = (Set<Object>)reuse;
+      result.clear();
+    } else {
+      result = new HashSet<Object>();
+    }
+    for(int i = 0; i < theset.size; i++) {
+      Object elem = this.getElementType().deserialize(null, iprot);
+      result.add(elem);
+    }
+    // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy
+    iprot.readSetEnd();
+    return result;
+  }
+
+  /** NOTE: Set is not supported by Hive yet.
+   *  The code uses ListObjectInspector right now. We need to change it to 
+   *  SetObjectInspector when that is done.
+   */
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot)
+  throws TException, SerDeException, NoSuchFieldException,
+  IllegalAccessException {
+
+    ListObjectInspector loi = (ListObjectInspector)oi;
+
+    Set<Object> set = (Set<Object>)o;
+    DynamicSerDeTypeBase mt = this.getElementType();
+    oprot.writeSetBegin(new TSet(mt.getType(),set.size()));
+    for(Object element: set) {
+      mt.serialize(element, loi.getListElementObjectInspector(), oprot);
+    }
+    // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy
+    oprot.writeSetEnd();
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType.*;
+
+public class DynamicSerDeTypeString extends DynamicSerDeTypeBase {
+
+  // production is: string
+
+  public DynamicSerDeTypeString(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypeString(thrift_grammar p, int i) {
+    super(p,i);
+  }
+  public Class getRealType() { return java.lang.String.class; }
+
+  public String toString() { return "string"; }
+
+  public String deserialize(TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    return String.valueOf(iprot.readString());
+  }
+
+  @Override
+  public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    return String.valueOf(iprot.readString());
+  }
+
+  public void serialize(Object s, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    oprot.writeString((String)s);
+  }
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    assert(oi.getCategory() == ObjectInspector.Category.PRIMITIVE);
+    assert(((PrimitiveObjectInspector)oi).getPrimitiveClass().equals(String.class));
+    oprot.writeString((String)o);
+  }
+
+  public byte getType() {
+    return TType.STRING;
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypedef.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypedef.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypedef.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypedef.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType.*;
+
+public class DynamicSerDeTypedef extends DynamicSerDeTypeBase {
+
+  // production is: typedef DefinitionType() this.name
+
+  private final static int FD_DEFINITION_TYPE = 0;
+
+  public DynamicSerDeTypedef(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypedef(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  private DynamicSerDeSimpleNode getDefinitionType() {
+    return (DynamicSerDeSimpleNode)this.jjtGetChild(FD_DEFINITION_TYPE);
+  }
+
+
+  public DynamicSerDeTypeBase getMyType() {
+    DynamicSerDeSimpleNode child = this.getDefinitionType();
+    DynamicSerDeTypeBase ret = (DynamicSerDeTypeBase)child.jjtGetChild(0);
+    return ret;
+  }
+
+  public String toString() {
+    String result = "typedef " + this.name + "(";
+    result += this.getDefinitionType().toString();
+    result += ")";
+    return result;
+  }
+
+  public byte getType() {
+    throw new RuntimeException("not implemented");
+  }
+
+  @Override
+  public Object deserialize(Object reuse, TProtocol iprot)
+  throws SerDeException, TException, IllegalAccessException {
+    throw new RuntimeException("not implemented");
+  }
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot)
+  throws TException, SerDeException, NoSuchFieldException,
+  IllegalAccessException {
+    throw new RuntimeException("not implemented");
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypei16 extends DynamicSerDeTypeBase {
+
+  public Class getRealType() { return Integer.valueOf(2).getClass(); }
+
+  // production is: i16
+
+  public DynamicSerDeTypei16(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypei16(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public String toString() { return "i16"; }
+
+  @Override
+  public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    return Integer.valueOf(iprot.readI16());
+  }
+
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    assert(oi.getCategory() == ObjectInspector.Category.PRIMITIVE);
+    assert(((PrimitiveObjectInspector)oi).getPrimitiveClass().equals(Short.class));
+    oprot.writeI16((Short)o);
+  }
+
+  public byte getType() {
+    return TType.I16;
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypei32 extends DynamicSerDeTypeBase {
+
+  // production is: i32
+
+  public DynamicSerDeTypei32(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypei32(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public String toString() { return "i32"; }
+
+  @Override
+  public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException {
+    return Integer.valueOf(iprot.readI32());
+  }
+
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    assert(oi.getCategory() == ObjectInspector.Category.PRIMITIVE);
+    assert(((PrimitiveObjectInspector)oi).getPrimitiveClass().equals(Integer.class));
+    oprot.writeI32((Integer)o);
+  }
+
+  public Class getRealType() { return java.lang.Integer.class; }
+  public Integer getRealTypeInstance() { return Integer.valueOf(0); }
+
+
+  public byte getType() {
+    return TType.I32;
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java?rev=706704&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java Tue Oct 21 11:11:05 2008
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+import com.facebook.thrift.TException;
+import com.facebook.thrift.TApplicationException;
+import com.facebook.thrift.protocol.*;
+import com.facebook.thrift.server.*;
+import com.facebook.thrift.transport.*;
+import java.util.*;
+import java.io.*;
+import org.apache.hadoop.hive.serde2.*;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+
+import java.lang.reflect.*;
+import com.facebook.thrift.protocol.TType;
+
+public class DynamicSerDeTypei64 extends DynamicSerDeTypeBase {
+
+  public Class getRealType() { return Long.valueOf(0).getClass(); }
+
+  // production is: i64
+
+  public DynamicSerDeTypei64(int i) {
+    super(i);
+  }
+  public DynamicSerDeTypei64(thrift_grammar p, int i) {
+    super(p,i);
+  }
+
+  public String toString() { return "i64"; }
+
+  @Override
+  public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
+    return Long.valueOf(iprot.readI64());
+  }
+
+  @Override
+  public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
+    assert(oi.getCategory() == ObjectInspector.Category.PRIMITIVE);
+    assert(((PrimitiveObjectInspector)oi).getPrimitiveClass().equals(Long.class));
+    oprot.writeI64((Long)o);
+  }
+
+  public byte getType() {
+    return TType.I64;
+  }
+}



Mime
View raw message