hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From dhr...@apache.org
Subject svn commit: r712905 [34/38] - in /hadoop/core/trunk: ./ src/contrib/hive/ src/contrib/hive/cli/src/java/org/apache/hadoop/hive/cli/ src/contrib/hive/common/src/java/org/apache/hadoop/hive/conf/ src/contrib/hive/conf/ src/contrib/hive/data/files/ src/co...
Date Tue, 11 Nov 2008 01:50:18 GMT
Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBase.java Mon Nov 10 17:50:06 2008
@@ -36,6 +36,11 @@
     super(p,i);
   }
 
+  public void initialize() {
+    // for base type, do nothing. Other types, like structs may initialize internal data
+    // structures.
+  }
+
   public Class getRealType() throws SerDeException {
     throw new SerDeException("Not implemented in base");
   }

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeBool.java Mon Nov 10 17:50:06 2008
@@ -45,7 +45,12 @@
   @Override
   public Object deserialize(Object reuse, TProtocol iprot)
       throws SerDeException, TException, IllegalAccessException {
-    return Boolean.valueOf(iprot.readBool());
+    boolean val = iprot.readBool();
+    if (val == false && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && 
+        ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) {
+      return null;
+    }
+    return Boolean.valueOf(val);
   }
 
   @Override
@@ -61,4 +66,8 @@
   public byte getType() {
     return TType.BOOL;
   }
+  
+  public Class getRealType() { return java.lang.Boolean.class; }
+  public Boolean getRealTypeInstance() { return Boolean.FALSE; }
+  
 }

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeByte.java Mon Nov 10 17:50:06 2008
@@ -41,10 +41,15 @@
   public String toString() { return "byte"; }
 
   public Byte deserialize(TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
-    return Byte.valueOf(iprot.readByte());
+    byte val = iprot.readByte();
+    if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && 
+        ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) {
+      return null;
+    }
+    return Byte.valueOf(val);
   }
   public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
-    return Byte.valueOf(iprot.readByte());
+    return deserialize(iprot);
   }
 
   public void serialize(Object s, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDefinition.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDefinition.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDefinition.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDefinition.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeTypeDefinition.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeTypeDefinition extends SimpleNode {
+  public DynamicSerDeTypeDefinition(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeTypeDefinition(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeDouble.java Mon Nov 10 17:50:06 2008
@@ -46,7 +46,12 @@
   public String toString() { return "double"; }
 
   public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
-    return Double.valueOf(iprot.readDouble());
+    double val = iprot.readDouble();
+    if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && 
+        ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) {
+      return null;
+    }
+    return Double.valueOf(val);
   }
 
   public void serialize(Object o, ObjectInspector oi, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
@@ -58,4 +63,7 @@
   public byte getType() {
     return TType.DOUBLE;
   }
+  
+  public Class getRealType() { return java.lang.Double.class; }
+  public Double getRealTypeInstance() { return Double.valueOf(0); }
 }

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeList.java Mon Nov 10 17:50:06 2008
@@ -59,6 +59,10 @@
   @Override
   public ArrayList<Object> deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
     TList thelist = iprot.readListBegin();
+    if (thelist == null) {
+      return null;
+    }
+
     ArrayList<Object> deserializeReuse;
     if (reuse != null) {
       deserializeReuse = (ArrayList<Object>)reuse;
@@ -89,17 +93,32 @@
     ObjectInspector elementObjectInspector = loi.getListElementObjectInspector();
     DynamicSerDeTypeBase mt = this.getElementType();
 
+    org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol nullProtocol = 
+      (oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)
+      ? (org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)oprot
+      : null;
+    
     if (o instanceof List) {
       List<?> list = (List<?>)o;
       oprot.writeListBegin(new TList(mt.getType(),list.size()));
       for (Object element: list) {
-        mt.serialize(element, elementObjectInspector, oprot);
+        if (element == null) {
+          assert(nullProtocol != null);
+          nullProtocol.writeNull();
+        } else {
+          mt.serialize(element, elementObjectInspector, oprot);
+        }
       }
     } else {
       Object[] list = (Object[])o;
       oprot.writeListBegin(new TList(mt.getType(),list.length));
       for (Object element: list) {
-        mt.serialize(element, elementObjectInspector, oprot);
+        if (element == null && nullProtocol != null) {
+          assert(nullProtocol != null);
+          nullProtocol.writeNull();
+        } else {
+          mt.serialize(element, elementObjectInspector, oprot);
+        }
       }
     }
     // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeMap.java Mon Nov 10 17:50:06 2008
@@ -86,7 +86,12 @@
       deserializeReuse = new HashMap<Object, Object>();
     }
     TMap themap = iprot.readMapBegin();
-    for(int i = 0; i < themap.size; i++) {
+    if (themap == null) {
+      return null;
+    }
+    // themap might be reused by the Protocol.
+    int mapSize = themap.size;
+    for(int i = 0; i < mapSize; i++) {
       Object key = this.getKeyType().deserialize(null, iprot);
       Object value = this.getValueType().deserialize(null, iprot);
       deserializeReuse.put(key,value);
@@ -97,6 +102,7 @@
     return deserializeReuse;
   }
 
+  TMap serializeMap = new TMap();
   @Override
   public void serialize(Object o, ObjectInspector oi, TProtocol oprot)
   throws TException, SerDeException, NoSuchFieldException,
@@ -104,19 +110,33 @@
     DynamicSerDeTypeBase keyType = this.getKeyType();
     DynamicSerDeTypeBase valueType = this.getValueType();
 
+    org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol nullProtocol = 
+      (oprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)
+      ? (org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)oprot
+      : null;
+    
     assert(oi.getCategory() == ObjectInspector.Category.MAP);
     MapObjectInspector moi = (MapObjectInspector)oi;
     ObjectInspector koi = moi.getMapKeyObjectInspector();
     ObjectInspector voi = moi.getMapValueObjectInspector();
 
     Map<?,?> map = moi.getMap(o);
-    oprot.writeMapBegin(new TMap(keyType.getType(),valueType.getType(),map.size()));
+    serializeMap.size = map.size();
+    serializeMap.keyType = keyType.getType();
+    serializeMap.valueType = valueType.getType();
+    oprot.writeMapBegin(serializeMap);
+    
     for(Iterator i = map.entrySet().iterator(); i.hasNext(); ) {
       Map.Entry it = (Map.Entry)i.next();
       Object key = it.getKey();
       Object value = it.getValue();
       keyType.serialize(key, koi, oprot);
-      valueType.serialize(value, voi, oprot);
+      if (value == null) {
+        assert(nullProtocol != null);
+        nullProtocol.writeNull();
+      } else {
+        valueType.serialize(value, voi, oprot);
+      }
     }
     // in theory, the below call isn't needed in non thrift_mode, but let's not get too crazy
     oprot.writeMapEnd();      

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeSet.java Mon Nov 10 17:50:06 2008
@@ -76,6 +76,9 @@
   public Object deserialize(Object reuse, TProtocol iprot)
   throws SerDeException, TException, IllegalAccessException {
     TSet theset = iprot.readSetBegin();
+    if (theset == null) {
+      return null;
+    }
     Set<Object> result;
     if (reuse != null) {
       result = (Set<Object>)reuse;
@@ -96,6 +99,7 @@
    *  The code uses ListObjectInspector right now. We need to change it to 
    *  SetObjectInspector when that is done.
    */
+  TSet tset = new TSet();
   @Override
   public void serialize(Object o, ObjectInspector oi, TProtocol oprot)
   throws TException, SerDeException, NoSuchFieldException,
@@ -105,7 +109,9 @@
 
     Set<Object> set = (Set<Object>)o;
     DynamicSerDeTypeBase mt = this.getElementType();
-    oprot.writeSetBegin(new TSet(mt.getType(),set.size()));
+    tset.elemType = mt.getType();
+    tset.size = set.size();
+    oprot.writeSetBegin(tset);
     for(Object element: set) {
       mt.serialize(element, loi.getListElementObjectInspector(), oprot);
     }

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypeString.java Mon Nov 10 17:50:06 2008
@@ -47,14 +47,14 @@
   public String toString() { return "string"; }
 
   public String deserialize(TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
-    return String.valueOf(iprot.readString());
+    return iprot.readString();
   }
 
   @Override
   public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
-    return String.valueOf(iprot.readString());
+    return iprot.readString();
   }
-
+  
   public void serialize(Object s, TProtocol oprot) throws TException, SerDeException, NoSuchFieldException,IllegalAccessException  {
     oprot.writeString((String)s);
   }

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei16.java Mon Nov 10 17:50:06 2008
@@ -49,7 +49,12 @@
 
   @Override
   public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
-    return Integer.valueOf(iprot.readI16());
+    int val = iprot.readI16();
+    if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && 
+        ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) {
+      return null;
+    }
+    return Integer.valueOf(val);
   }
 
   @Override

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei32.java Mon Nov 10 17:50:06 2008
@@ -47,7 +47,12 @@
 
   @Override
   public Object deserialize(Object reuse, TProtocol iprot) throws SerDeException, TException, IllegalAccessException {
-    return Integer.valueOf(iprot.readI32());
+    int val = iprot.readI32();
+    if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && 
+        ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) {
+      return null;
+    }
+    return Integer.valueOf(val);
   }
 
   @Override

Modified: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java?rev=712905&r1=712904&r2=712905&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java (original)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeTypei64.java Mon Nov 10 17:50:06 2008
@@ -49,7 +49,12 @@
 
   @Override
   public Object deserialize(Object reuse, TProtocol iprot)  throws SerDeException, TException, IllegalAccessException {
-    return Long.valueOf(iprot.readI64());
+    long val = iprot.readI64();
+    if (val == 0 && iprot instanceof org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol && 
+        ((org.apache.hadoop.hive.serde2.thrift.WriteNullsProtocol)iprot).lastPrimitiveWasNull()) {
+      return null;
+    }
+    return Long.valueOf(val);
   }
 
   @Override

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeUnflagArgs.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeUnflagArgs.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeUnflagArgs.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeUnflagArgs.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeUnflagArgs.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeUnflagArgs extends SimpleNode {
+  public DynamicSerDeUnflagArgs(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeUnflagArgs(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeXception.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeXception.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeXception.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/DynamicSerDeXception.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,14 @@
+/* Generated By:JJTree: Do not edit this line. DynamicSerDeXception.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class DynamicSerDeXception extends SimpleNode {
+  public DynamicSerDeXception(int id) {
+    super(id);
+  }
+
+  public DynamicSerDeXception(thrift_grammar p, int id) {
+    super(p, id);
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,123 @@
+/* Generated By:JJTree: Do not edit this line. /home/pwyckoff/projects/hadoop/trunk/VENDOR/hadoop-0.17/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/JJTthrift_grammarState.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+class JJTthrift_grammarState {
+  private java.util.Stack nodes;
+  private java.util.Stack marks;
+
+  private int sp;		// number of nodes on stack
+  private int mk;		// current mark
+  private boolean node_created;
+
+  JJTthrift_grammarState() {
+    nodes = new java.util.Stack();
+    marks = new java.util.Stack();
+    sp = 0;
+    mk = 0;
+  }
+
+  /* Determines whether the current node was actually closed and
+     pushed.  This should only be called in the final user action of a
+     node scope.  */
+  boolean nodeCreated() {
+    return node_created;
+  }
+
+  /* Call this to reinitialize the node stack.  It is called
+     automatically by the parser's ReInit() method. */
+  void reset() {
+    nodes.removeAllElements();
+    marks.removeAllElements();
+    sp = 0;
+    mk = 0;
+  }
+
+  /* Returns the root node of the AST.  It only makes sense to call
+     this after a successful parse. */
+  Node rootNode() {
+    return (Node)nodes.elementAt(0);
+  }
+
+  /* Pushes a node on to the stack. */
+  void pushNode(Node n) {
+    nodes.push(n);
+    ++sp;
+  }
+
+  /* Returns the node on the top of the stack, and remove it from the
+     stack.  */
+  Node popNode() {
+    if (--sp < mk) {
+      mk = ((Integer)marks.pop()).intValue();
+    }
+    return (Node)nodes.pop();
+  }
+
+  /* Returns the node currently on the top of the stack. */
+  Node peekNode() {
+    return (Node)nodes.peek();
+  }
+
+  /* Returns the number of children on the stack in the current node
+     scope. */
+  int nodeArity() {
+    return sp - mk;
+  }
+
+
+  void clearNodeScope(Node n) {
+    while (sp > mk) {
+      popNode();
+    }
+    mk = ((Integer)marks.pop()).intValue();
+  }
+
+
+  void openNodeScope(Node n) {
+    marks.push(new Integer(mk));
+    mk = sp;
+    n.jjtOpen();
+  }
+
+
+  /* A definite node is constructed from a specified number of
+     children.  That number of nodes are popped from the stack and
+     made the children of the definite node.  Then the definite node
+     is pushed on to the stack. */
+  void closeNodeScope(Node n, int num) {
+    mk = ((Integer)marks.pop()).intValue();
+    while (num-- > 0) {
+      Node c = popNode();
+      c.jjtSetParent(n);
+      n.jjtAddChild(c, num);
+    }
+    n.jjtClose();
+    pushNode(n);
+    node_created = true;
+  }
+
+
+  /* A conditional node is constructed if its condition is true.  All
+     the nodes that have been pushed since the node was opened are
+     made children of the the conditional node, which is then pushed
+     on to the stack.  If the condition is false the node is not
+     constructed and they are left on the stack. */
+  void closeNodeScope(Node n, boolean condition) {
+    if (condition) {
+      int a = nodeArity();
+      mk = ((Integer)marks.pop()).intValue();
+      while (a-- > 0) {
+	Node c = popNode();
+	c.jjtSetParent(n);
+	n.jjtAddChild(c, a);
+      }
+      n.jjtClose();
+      pushNode(n);
+      node_created = true;
+    } else {
+      mk = ((Integer)marks.pop()).intValue();
+      node_created = false;
+    }
+  }
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Node.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Node.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Node.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Node.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,34 @@
+/* Generated By:JJTree: Do not edit this line. Node.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+/* All AST nodes must implement this interface.  It provides basic
+   machinery for constructing the parent and child relationships
+   between nodes. */
+
+public interface Node {
+
+  /** This method is called after the node has been made the current
+    node.  It indicates that child nodes can now be added to it. */
+  public void jjtOpen();
+
+  /** This method is called after all the child nodes have been
+    added. */
+  public void jjtClose();
+
+  /** This pair of methods are used to inform the node of its
+    parent. */
+  public void jjtSetParent(Node n);
+  public Node jjtGetParent();
+
+  /** This method tells the node to add its argument to the node's
+    list of children.  */
+  public void jjtAddChild(Node n, int i);
+
+  /** This method returns a child node.  The children are numbered
+     from zero, left to right. */
+  public Node jjtGetChild(int i);
+
+  /** Return the number of children the node has. */
+  public int jjtGetNumChildren();
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/ParseException.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/ParseException.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/ParseException.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/ParseException.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,192 @@
+/* Generated By:JavaCC: Do not edit this line. ParseException.java Version 3.0 */
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+/**
+ * This exception is thrown when parse errors are encountered.
+ * You can explicitly create objects of this exception type by
+ * calling the method generateParseException in the generated
+ * parser.
+ *
+ * You can modify this class to customize your error reporting
+ * mechanisms so long as you retain the public fields.
+ */
+public class ParseException extends Exception {
+
+  /**
+   * This constructor is used by the method "generateParseException"
+   * in the generated parser.  Calling this constructor generates
+   * a new object of this type with the fields "currentToken",
+   * "expectedTokenSequences", and "tokenImage" set.  The boolean
+   * flag "specialConstructor" is also set to true to indicate that
+   * this constructor was used to create this object.
+   * This constructor calls its super class with the empty string
+   * to force the "toString" method of parent class "Throwable" to
+   * print the error message in the form:
+   *     ParseException: <result of getMessage>
+   */
+  public ParseException(Token currentTokenVal,
+                        int[][] expectedTokenSequencesVal,
+                        String[] tokenImageVal
+                       )
+  {
+    super("");
+    specialConstructor = true;
+    currentToken = currentTokenVal;
+    expectedTokenSequences = expectedTokenSequencesVal;
+    tokenImage = tokenImageVal;
+  }
+
+  /**
+   * The following constructors are for use by you for whatever
+   * purpose you can think of.  Constructing the exception in this
+   * manner makes the exception behave in the normal way - i.e., as
+   * documented in the class "Throwable".  The fields "errorToken",
+   * "expectedTokenSequences", and "tokenImage" do not contain
+   * relevant information.  The JavaCC generated code does not use
+   * these constructors.
+   */
+
+  public ParseException() {
+    super();
+    specialConstructor = false;
+  }
+
+  public ParseException(String message) {
+    super(message);
+    specialConstructor = false;
+  }
+
+  /**
+   * This variable determines which constructor was used to create
+   * this object and thereby affects the semantics of the
+   * "getMessage" method (see below).
+   */
+  protected boolean specialConstructor;
+
+  /**
+   * This is the last token that has been consumed successfully.  If
+   * this object has been created due to a parse error, the token
+   * followng this token will (therefore) be the first error token.
+   */
+  public Token currentToken;
+
+  /**
+   * Each entry in this array is an array of integers.  Each array
+   * of integers represents a sequence of tokens (by their ordinal
+   * values) that is expected at this point of the parse.
+   */
+  public int[][] expectedTokenSequences;
+
+  /**
+   * This is a reference to the "tokenImage" array of the generated
+   * parser within which the parse error occurred.  This array is
+   * defined in the generated ...Constants interface.
+   */
+  public String[] tokenImage;
+
+  /**
+   * This method has the standard behavior when this object has been
+   * created using the standard constructors.  Otherwise, it uses
+   * "currentToken" and "expectedTokenSequences" to generate a parse
+   * error message and returns it.  If this object has been created
+   * due to a parse error, and you do not catch it (it gets thrown
+   * from the parser), then this method is called during the printing
+   * of the final stack trace, and hence the correct error message
+   * gets displayed.
+   */
+  public String getMessage() {
+    if (!specialConstructor) {
+      return super.getMessage();
+    }
+    StringBuffer expected = new StringBuffer();
+    int maxSize = 0;
+    for (int i = 0; i < expectedTokenSequences.length; i++) {
+      if (maxSize < expectedTokenSequences[i].length) {
+        maxSize = expectedTokenSequences[i].length;
+      }
+      for (int j = 0; j < expectedTokenSequences[i].length; j++) {
+        expected.append(tokenImage[expectedTokenSequences[i][j]]).append(" ");
+      }
+      if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
+        expected.append("...");
+      }
+      expected.append(eol).append("    ");
+    }
+    String retval = "Encountered \"";
+    Token tok = currentToken.next;
+    for (int i = 0; i < maxSize; i++) {
+      if (i != 0) retval += " ";
+      if (tok.kind == 0) {
+        retval += tokenImage[0];
+        break;
+      }
+      retval += add_escapes(tok.image);
+      tok = tok.next; 
+    }
+    retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
+    retval += "." + eol;
+    if (expectedTokenSequences.length == 1) {
+      retval += "Was expecting:" + eol + "    ";
+    } else {
+      retval += "Was expecting one of:" + eol + "    ";
+    }
+    retval += expected.toString();
+    return retval;
+  }
+
+  /**
+   * The end of line string for this machine.
+   */
+  protected String eol = System.getProperty("line.separator", "\n");
+ 
+  /**
+   * Used to convert raw characters to their escaped version
+   * when these raw version cannot be used as part of an ASCII
+   * string literal.
+   */
+  protected String add_escapes(String str) {
+      StringBuffer retval = new StringBuffer();
+      char ch;
+      for (int i = 0; i < str.length(); i++) {
+        switch (str.charAt(i))
+        {
+           case 0 :
+              continue;
+           case '\b':
+              retval.append("\\b");
+              continue;
+           case '\t':
+              retval.append("\\t");
+              continue;
+           case '\n':
+              retval.append("\\n");
+              continue;
+           case '\f':
+              retval.append("\\f");
+              continue;
+           case '\r':
+              retval.append("\\r");
+              continue;
+           case '\"':
+              retval.append("\\\"");
+              continue;
+           case '\'':
+              retval.append("\\\'");
+              continue;
+           case '\\':
+              retval.append("\\\\");
+              continue;
+           default:
+              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+                 String s = "0000" + Integer.toString(ch, 16);
+                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+              } else {
+                 retval.append(ch);
+              }
+              continue;
+        }
+      }
+      return retval.toString();
+   }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleCharStream.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleCharStream.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleCharStream.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleCharStream.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,439 @@
+/* Generated By:JavaCC: Do not edit this line. SimpleCharStream.java Version 4.0 */
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+/**
+ * An implementation of interface CharStream, where the stream is assumed to
+ * contain only ASCII characters (without unicode processing).
+ */
+
+public class SimpleCharStream
+{
+  public static final boolean staticFlag = false;
+  int bufsize;
+  int available;
+  int tokenBegin;
+  public int bufpos = -1;
+  protected int bufline[];
+  protected int bufcolumn[];
+
+  protected int column = 0;
+  protected int line = 1;
+
+  protected boolean prevCharIsCR = false;
+  protected boolean prevCharIsLF = false;
+
+  protected java.io.Reader inputStream;
+
+  protected char[] buffer;
+  protected int maxNextCharInd = 0;
+  protected int inBuf = 0;
+  protected int tabSize = 8;
+
+  protected void setTabSize(int i) { tabSize = i; }
+  protected int getTabSize(int i) { return tabSize; }
+
+
+  protected void ExpandBuff(boolean wrapAround)
+  {
+     char[] newbuffer = new char[bufsize + 2048];
+     int newbufline[] = new int[bufsize + 2048];
+     int newbufcolumn[] = new int[bufsize + 2048];
+
+     try
+     {
+        if (wrapAround)
+        {
+           System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+           System.arraycopy(buffer, 0, newbuffer,
+                                             bufsize - tokenBegin, bufpos);
+           buffer = newbuffer;
+
+           System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+           System.arraycopy(bufline, 0, newbufline, bufsize - tokenBegin, bufpos);
+           bufline = newbufline;
+
+           System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+           System.arraycopy(bufcolumn, 0, newbufcolumn, bufsize - tokenBegin, bufpos);
+           bufcolumn = newbufcolumn;
+
+           maxNextCharInd = (bufpos += (bufsize - tokenBegin));
+        }
+        else
+        {
+           System.arraycopy(buffer, tokenBegin, newbuffer, 0, bufsize - tokenBegin);
+           buffer = newbuffer;
+
+           System.arraycopy(bufline, tokenBegin, newbufline, 0, bufsize - tokenBegin);
+           bufline = newbufline;
+
+           System.arraycopy(bufcolumn, tokenBegin, newbufcolumn, 0, bufsize - tokenBegin);
+           bufcolumn = newbufcolumn;
+
+           maxNextCharInd = (bufpos -= tokenBegin);
+        }
+     }
+     catch (Throwable t)
+     {
+        throw new Error(t.getMessage());
+     }
+
+
+     bufsize += 2048;
+     available = bufsize;
+     tokenBegin = 0;
+  }
+
+  protected void FillBuff() throws java.io.IOException
+  {
+     if (maxNextCharInd == available)
+     {
+        if (available == bufsize)
+        {
+           if (tokenBegin > 2048)
+           {
+              bufpos = maxNextCharInd = 0;
+              available = tokenBegin;
+           }
+           else if (tokenBegin < 0)
+              bufpos = maxNextCharInd = 0;
+           else
+              ExpandBuff(false);
+        }
+        else if (available > tokenBegin)
+           available = bufsize;
+        else if ((tokenBegin - available) < 2048)
+           ExpandBuff(true);
+        else
+           available = tokenBegin;
+     }
+
+     int i;
+     try {
+        if ((i = inputStream.read(buffer, maxNextCharInd,
+                                    available - maxNextCharInd)) == -1)
+        {
+           inputStream.close();
+           throw new java.io.IOException();
+        }
+        else
+           maxNextCharInd += i;
+        return;
+     }
+     catch(java.io.IOException e) {
+        --bufpos;
+        backup(0);
+        if (tokenBegin == -1)
+           tokenBegin = bufpos;
+        throw e;
+     }
+  }
+
+  public char BeginToken() throws java.io.IOException
+  {
+     tokenBegin = -1;
+     char c = readChar();
+     tokenBegin = bufpos;
+
+     return c;
+  }
+
+  protected void UpdateLineColumn(char c)
+  {
+     column++;
+
+     if (prevCharIsLF)
+     {
+        prevCharIsLF = false;
+        line += (column = 1);
+     }
+     else if (prevCharIsCR)
+     {
+        prevCharIsCR = false;
+        if (c == '\n')
+        {
+           prevCharIsLF = true;
+        }
+        else
+           line += (column = 1);
+     }
+
+     switch (c)
+     {
+        case '\r' :
+           prevCharIsCR = true;
+           break;
+        case '\n' :
+           prevCharIsLF = true;
+           break;
+        case '\t' :
+           column--;
+           column += (tabSize - (column % tabSize));
+           break;
+        default :
+           break;
+     }
+
+     bufline[bufpos] = line;
+     bufcolumn[bufpos] = column;
+  }
+
+  public char readChar() throws java.io.IOException
+  {
+     if (inBuf > 0)
+     {
+        --inBuf;
+
+        if (++bufpos == bufsize)
+           bufpos = 0;
+
+        return buffer[bufpos];
+     }
+
+     if (++bufpos >= maxNextCharInd)
+        FillBuff();
+
+     char c = buffer[bufpos];
+
+     UpdateLineColumn(c);
+     return (c);
+  }
+
+  /**
+   * @deprecated 
+   * @see #getEndColumn
+   */
+
+  public int getColumn() {
+     return bufcolumn[bufpos];
+  }
+
+  /**
+   * @deprecated 
+   * @see #getEndLine
+   */
+
+  public int getLine() {
+     return bufline[bufpos];
+  }
+
+  public int getEndColumn() {
+     return bufcolumn[bufpos];
+  }
+
+  public int getEndLine() {
+     return bufline[bufpos];
+  }
+
+  public int getBeginColumn() {
+     return bufcolumn[tokenBegin];
+  }
+
+  public int getBeginLine() {
+     return bufline[tokenBegin];
+  }
+
+  public void backup(int amount) {
+
+    inBuf += amount;
+    if ((bufpos -= amount) < 0)
+       bufpos += bufsize;
+  }
+
+  public SimpleCharStream(java.io.Reader dstream, int startline,
+  int startcolumn, int buffersize)
+  {
+    inputStream = dstream;
+    line = startline;
+    column = startcolumn - 1;
+
+    available = bufsize = buffersize;
+    buffer = new char[buffersize];
+    bufline = new int[buffersize];
+    bufcolumn = new int[buffersize];
+  }
+
+  public SimpleCharStream(java.io.Reader dstream, int startline,
+                          int startcolumn)
+  {
+     this(dstream, startline, startcolumn, 4096);
+  }
+
+  public SimpleCharStream(java.io.Reader dstream)
+  {
+     this(dstream, 1, 1, 4096);
+  }
+  public void ReInit(java.io.Reader dstream, int startline,
+  int startcolumn, int buffersize)
+  {
+    inputStream = dstream;
+    line = startline;
+    column = startcolumn - 1;
+
+    if (buffer == null || buffersize != buffer.length)
+    {
+      available = bufsize = buffersize;
+      buffer = new char[buffersize];
+      bufline = new int[buffersize];
+      bufcolumn = new int[buffersize];
+    }
+    prevCharIsLF = prevCharIsCR = false;
+    tokenBegin = inBuf = maxNextCharInd = 0;
+    bufpos = -1;
+  }
+
+  public void ReInit(java.io.Reader dstream, int startline,
+                     int startcolumn)
+  {
+     ReInit(dstream, startline, startcolumn, 4096);
+  }
+
+  public void ReInit(java.io.Reader dstream)
+  {
+     ReInit(dstream, 1, 1, 4096);
+  }
+  public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,
+  int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+  {
+     this(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+  }
+
+  public SimpleCharStream(java.io.InputStream dstream, int startline,
+  int startcolumn, int buffersize)
+  {
+     this(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+  }
+
+  public SimpleCharStream(java.io.InputStream dstream, String encoding, int startline,
+                          int startcolumn) throws java.io.UnsupportedEncodingException
+  {
+     this(dstream, encoding, startline, startcolumn, 4096);
+  }
+
+  public SimpleCharStream(java.io.InputStream dstream, int startline,
+                          int startcolumn)
+  {
+     this(dstream, startline, startcolumn, 4096);
+  }
+
+  public SimpleCharStream(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+  {
+     this(dstream, encoding, 1, 1, 4096);
+  }
+
+  public SimpleCharStream(java.io.InputStream dstream)
+  {
+     this(dstream, 1, 1, 4096);
+  }
+
+  public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+                          int startcolumn, int buffersize) throws java.io.UnsupportedEncodingException
+  {
+     ReInit(encoding == null ? new java.io.InputStreamReader(dstream) : new java.io.InputStreamReader(dstream, encoding), startline, startcolumn, buffersize);
+  }
+
+  public void ReInit(java.io.InputStream dstream, int startline,
+                          int startcolumn, int buffersize)
+  {
+     ReInit(new java.io.InputStreamReader(dstream), startline, startcolumn, buffersize);
+  }
+
+  public void ReInit(java.io.InputStream dstream, String encoding) throws java.io.UnsupportedEncodingException
+  {
+     ReInit(dstream, encoding, 1, 1, 4096);
+  }
+
+  public void ReInit(java.io.InputStream dstream)
+  {
+     ReInit(dstream, 1, 1, 4096);
+  }
+  public void ReInit(java.io.InputStream dstream, String encoding, int startline,
+                     int startcolumn) throws java.io.UnsupportedEncodingException
+  {
+     ReInit(dstream, encoding, startline, startcolumn, 4096);
+  }
+  public void ReInit(java.io.InputStream dstream, int startline,
+                     int startcolumn)
+  {
+     ReInit(dstream, startline, startcolumn, 4096);
+  }
+  public String GetImage()
+  {
+     if (bufpos >= tokenBegin)
+        return new String(buffer, tokenBegin, bufpos - tokenBegin + 1);
+     else
+        return new String(buffer, tokenBegin, bufsize - tokenBegin) +
+                              new String(buffer, 0, bufpos + 1);
+  }
+
+  public char[] GetSuffix(int len)
+  {
+     char[] ret = new char[len];
+
+     if ((bufpos + 1) >= len)
+        System.arraycopy(buffer, bufpos - len + 1, ret, 0, len);
+     else
+     {
+        System.arraycopy(buffer, bufsize - (len - bufpos - 1), ret, 0,
+                                                          len - bufpos - 1);
+        System.arraycopy(buffer, 0, ret, len - bufpos - 1, bufpos + 1);
+     }
+
+     return ret;
+  }
+
+  public void Done()
+  {
+     buffer = null;
+     bufline = null;
+     bufcolumn = null;
+  }
+
+  /**
+   * Method to adjust line and column numbers for the start of a token.
+   */
+  public void adjustBeginLineColumn(int newLine, int newCol)
+  {
+     int start = tokenBegin;
+     int len;
+
+     if (bufpos >= tokenBegin)
+     {
+        len = bufpos - tokenBegin + inBuf + 1;
+     }
+     else
+     {
+        len = bufsize - tokenBegin + bufpos + 1 + inBuf;
+     }
+
+     int i = 0, j = 0, k = 0;
+     int nextColDiff = 0, columnDiff = 0;
+
+     while (i < len &&
+            bufline[j = start % bufsize] == bufline[k = ++start % bufsize])
+     {
+        bufline[j] = newLine;
+        nextColDiff = columnDiff + bufcolumn[k] - bufcolumn[j];
+        bufcolumn[j] = newCol + columnDiff;
+        columnDiff = nextColDiff;
+        i++;
+     } 
+
+     if (i < len)
+     {
+        bufline[j] = newLine++;
+        bufcolumn[j] = newCol + columnDiff;
+
+        while (i++ < len)
+        {
+           if (bufline[j = start % bufsize] != bufline[++start % bufsize])
+              bufline[j] = newLine++;
+           else
+              bufline[j] = newLine;
+        }
+     }
+
+     line = bufline[j];
+     column = bufcolumn[j];
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleNode.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleNode.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleNode.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/SimpleNode.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,72 @@
+/* Generated By:JJTree: Do not edit this line. SimpleNode.java */
+
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class SimpleNode implements Node {
+  protected Node parent;
+  protected Node[] children;
+  protected int id;
+  protected thrift_grammar parser;
+
+  public SimpleNode(int i) {
+    id = i;
+  }
+
+  public SimpleNode(thrift_grammar p, int i) {
+    this(i);
+    parser = p;
+  }
+
+  public void jjtOpen() {
+  }
+
+  public void jjtClose() {
+  }
+  
+  public void jjtSetParent(Node n) { parent = n; }
+  public Node jjtGetParent() { return parent; }
+
+  public void jjtAddChild(Node n, int i) {
+    if (children == null) {
+      children = new Node[i + 1];
+    } else if (i >= children.length) {
+      Node c[] = new Node[i + 1];
+      System.arraycopy(children, 0, c, 0, children.length);
+      children = c;
+    }
+    children[i] = n;
+  }
+
+  public Node jjtGetChild(int i) {
+    return children[i];
+  }
+
+  public int jjtGetNumChildren() {
+    return (children == null) ? 0 : children.length;
+  }
+
+  /* You can override these two methods in subclasses of SimpleNode to
+     customize the way the node appears when the tree is dumped.  If
+     your output uses more than one line you should override
+     toString(String), otherwise overriding toString() is probably all
+     you need to do. */
+
+  public String toString() { return thrift_grammarTreeConstants.jjtNodeName[id]; }
+  public String toString(String prefix) { return prefix + toString(); }
+
+  /* Override this method if you want to customize how the node dumps
+     out its children. */
+
+  public void dump(String prefix) {
+    System.out.println(toString(prefix));
+    if (children != null) {
+      for (int i = 0; i < children.length; ++i) {
+	SimpleNode n = (SimpleNode)children[i];
+	if (n != null) {
+	  n.dump(prefix + " ");
+	}
+      }
+    }
+  }
+}
+

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Token.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Token.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Token.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/Token.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,81 @@
+/* Generated By:JavaCC: Do not edit this line. Token.java Version 3.0 */
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+/**
+ * Describes the input token stream.
+ */
+
+public class Token {
+
+  /**
+   * An integer that describes the kind of this token.  This numbering
+   * system is determined by JavaCCParser, and a table of these numbers is
+   * stored in the file ...Constants.java.
+   */
+  public int kind;
+
+  /**
+   * beginLine and beginColumn describe the position of the first character
+   * of this token; endLine and endColumn describe the position of the
+   * last character of this token.
+   */
+  public int beginLine, beginColumn, endLine, endColumn;
+
+  /**
+   * The string image of the token.
+   */
+  public String image;
+
+  /**
+   * A reference to the next regular (non-special) token from the input
+   * stream.  If this is the last token from the input stream, or if the
+   * token manager has not read tokens beyond this one, this field is
+   * set to null.  This is true only if this token is also a regular
+   * token.  Otherwise, see below for a description of the contents of
+   * this field.
+   */
+  public Token next;
+
+  /**
+   * This field is used to access special tokens that occur prior to this
+   * token, but after the immediately preceding regular (non-special) token.
+   * If there are no such special tokens, this field is set to null.
+   * When there are more than one such special token, this field refers
+   * to the last of these special tokens, which in turn refers to the next
+   * previous special token through its specialToken field, and so on
+   * until the first special token (whose specialToken field is null).
+   * The next fields of special tokens refer to other special tokens that
+   * immediately follow it (without an intervening regular token).  If there
+   * is no such token, this field is null.
+   */
+  public Token specialToken;
+
+  /**
+   * Returns the image.
+   */
+  public String toString()
+  {
+     return image;
+  }
+
+  /**
+   * Returns a new Token object, by default. However, if you want, you
+   * can create and return subclass objects based on the value of ofKind.
+   * Simply add the cases to the switch for all those special cases.
+   * For example, if you have a subclass of Token called IDToken that
+   * you want to create if ofKind is ID, simlpy add something like :
+   *
+   *    case MyParserConstants.ID : return new IDToken();
+   *
+   * to the following switch statement. Then you can cast matchedToken
+   * variable to the appropriate type and use it in your lexical actions.
+   */
+  public static final Token newToken(int ofKind)
+  {
+     switch(ofKind)
+     {
+       default : return new Token();
+     }
+  }
+
+}

Added: hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/TokenMgrError.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/TokenMgrError.java?rev=712905&view=auto
==============================================================================
--- hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/TokenMgrError.java (added)
+++ hadoop/core/trunk/src/contrib/hive/serde/src/java/org/apache/hadoop/hive/serde2/dynamic_type/TokenMgrError.java Mon Nov 10 17:50:06 2008
@@ -0,0 +1,133 @@
+/* Generated By:JavaCC: Do not edit this line. TokenMgrError.java Version 3.0 */
+package org.apache.hadoop.hive.serde2.dynamic_type;
+
+public class TokenMgrError extends Error
+{
+   /*
+    * Ordinals for various reasons why an Error of this type can be thrown.
+    */
+
+   /**
+    * Lexical error occured.
+    */
+   static final int LEXICAL_ERROR = 0;
+
+   /**
+    * An attempt wass made to create a second instance of a static token manager.
+    */
+   static final int STATIC_LEXER_ERROR = 1;
+
+   /**
+    * Tried to change to an invalid lexical state.
+    */
+   static final int INVALID_LEXICAL_STATE = 2;
+
+   /**
+    * Detected (and bailed out of) an infinite loop in the token manager.
+    */
+   static final int LOOP_DETECTED = 3;
+
+   /**
+    * Indicates the reason why the exception is thrown. It will have
+    * one of the above 4 values.
+    */
+   int errorCode;
+
+   /**
+    * Replaces unprintable characters by their espaced (or unicode escaped)
+    * equivalents in the given string
+    */
+   protected static final String addEscapes(String str) {
+      StringBuffer retval = new StringBuffer();
+      char ch;
+      for (int i = 0; i < str.length(); i++) {
+        switch (str.charAt(i))
+        {
+           case 0 :
+              continue;
+           case '\b':
+              retval.append("\\b");
+              continue;
+           case '\t':
+              retval.append("\\t");
+              continue;
+           case '\n':
+              retval.append("\\n");
+              continue;
+           case '\f':
+              retval.append("\\f");
+              continue;
+           case '\r':
+              retval.append("\\r");
+              continue;
+           case '\"':
+              retval.append("\\\"");
+              continue;
+           case '\'':
+              retval.append("\\\'");
+              continue;
+           case '\\':
+              retval.append("\\\\");
+              continue;
+           default:
+              if ((ch = str.charAt(i)) < 0x20 || ch > 0x7e) {
+                 String s = "0000" + Integer.toString(ch, 16);
+                 retval.append("\\u" + s.substring(s.length() - 4, s.length()));
+              } else {
+                 retval.append(ch);
+              }
+              continue;
+        }
+      }
+      return retval.toString();
+   }
+
+   /**
+    * Returns a detailed message for the Error when it is thrown by the
+    * token manager to indicate a lexical error.
+    * Parameters : 
+    *    EOFSeen     : indicates if EOF caused the lexicl error
+    *    curLexState : lexical state in which this error occured
+    *    errorLine   : line number when the error occured
+    *    errorColumn : column number when the error occured
+    *    errorAfter  : prefix that was seen before this error occured
+    *    curchar     : the offending character
+    * Note: You can customize the lexical error message by modifying this method.
+    */
+   protected static String LexicalError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar) {
+      return("Lexical error at line " +
+           errorLine + ", column " +
+           errorColumn + ".  Encountered: " +
+           (EOFSeen ? "<EOF> " : ("\"" + addEscapes(String.valueOf(curChar)) + "\"") + " (" + (int)curChar + "), ") +
+           "after : \"" + addEscapes(errorAfter) + "\"");
+   }
+
+   /**
+    * You can also modify the body of this method to customize your error messages.
+    * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+    * of end-users concern, so you can return something like : 
+    *
+    *     "Internal Error : Please file a bug report .... "
+    *
+    * from this method for such cases in the release version of your parser.
+    */
+   public String getMessage() {
+      return super.getMessage();
+   }
+
+   /*
+    * Constructors of various flavors follow.
+    */
+
+   public TokenMgrError() {
+   }
+
+   public TokenMgrError(String message, int reason) {
+      super(message);
+      errorCode = reason;
+   }
+
+   public TokenMgrError(boolean EOFSeen, int lexState, int errorLine, int errorColumn, String errorAfter, char curChar, int reason) {
+      this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
+   }
+}



Mime
View raw message