hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r513122 [1/3] - in /lucene/hadoop/trunk: ./ src/c++/librecordio/ src/java/org/apache/hadoop/io/ src/java/org/apache/hadoop/record/ src/java/org/apache/hadoop/record/compiler/ src/java/org/apache/hadoop/record/compiler/ant/ src/java/org/apac...
Date Thu, 01 Mar 2007 03:47:29 GMT
Author: cutting
Date: Wed Feb 28 19:47:27 2007
New Revision: 513122

URL: http://svn.apache.org/viewvc?view=rev&rev=513122
Log:
HADOOP-941.  Enhance record facility.  Contributed by Milind.

Added:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Buffer.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CGenerator.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeBuffer.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeGenerator.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/FromCpp.java
      - copied, changed from r513042, lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/FromCpp.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/TestBuffer.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/TestRecordIO.java
      - copied, changed from r513042, lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/TestRecordIO.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/TestRecordMR.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/TestRecordWritable.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/ToCpp.java
      - copied, changed from r513042, lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/ToCpp.java
Removed:
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/test/
Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/c++/librecordio/binarchive.cc
    lucene/hadoop/trunk/src/c++/librecordio/binarchive.hh
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableComparator.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableUtils.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFloat.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JInt.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JLong.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JString.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JType.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JVector.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JavaGenerator.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/ant/RccTask.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/Rcc.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/generated/rcc.jj
    lucene/hadoop/trunk/src/test/ddl/buffer.jr
    lucene/hadoop/trunk/src/test/ddl/int.jr
    lucene/hadoop/trunk/src/test/ddl/string.jr
    lucene/hadoop/trunk/src/test/ddl/test.jr

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Feb 28 19:47:27 2007
@@ -168,6 +168,9 @@
 50. HADOOP-1020.  Fix a bug in Path resolution, and a with unit tests
     on Windows.  (cutting)
 
+51. HADOOP-941.  Enhance record facility.
+    (Milind Bhandarkar via cutting)
+
 
 Release 0.11.2 - 2007-02-16
 

Modified: lucene/hadoop/trunk/src/c++/librecordio/binarchive.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/binarchive.cc?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/binarchive.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/binarchive.cc Wed Feb 28 19:47:27 2007
@@ -17,6 +17,8 @@
  */
 
 #include "binarchive.hh"
+#include <rpc/xdr.h>
+
 
 using namespace hadoop;
 
@@ -77,16 +79,20 @@
     t = b;
     return;
   }
-  b = (b < -124) ? -(b + 124) : -(b + 120);
+  bool isNegative = (b < -124);
+  b = isNegative ? -(b + 124) : -(b + 120);
   uint8_t barr[b];
   if (b != stream.read(barr, b)) {
-    throw new IOException("Error deserializing long");
+    throw new IOException("Error deserializing int");
   }
   t = 0;
   for (int idx = 0; idx < b; idx++) {
     t = t << 8;
     t |= (barr[idx] & 0xFF);
   }
+  if (isNegative) {
+    t |= 0x80000000;
+  }
 }
 
 static void serializeLong(int64_t t, OutStream& stream)
@@ -131,7 +137,8 @@
     t = b;
     return;
   }
-  b = (b < -120) ? -(b + 120) : -(b + 112);
+  bool isNegative = (b < -120);
+  b = isNegative ? -(b + 120) : -(b + 112);
   uint8_t barr[b];
   if (b != stream.read(barr, b)) {
     throw new IOException("Error deserializing long.");
@@ -140,6 +147,9 @@
   for (int idx = 0; idx < b; idx++) {
     t = t << 8;
     t |= (barr[idx] & 0xFF);
+  }
+  if (isNegative) {
+    t |= 0x8000000000000000L;
   }
 }
 

Modified: lucene/hadoop/trunk/src/c++/librecordio/binarchive.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/binarchive.hh?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/binarchive.hh (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/binarchive.hh Wed Feb 28 19:47:27 2007
@@ -20,7 +20,6 @@
 #define BINARCHIVE_HH_
 
 #include "recordio.hh"
-#include <rpc/xdr.h>
 
 namespace hadoop {
 

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableComparator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableComparator.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableComparator.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableComparator.java Wed Feb 28 19:47:27 2007
@@ -177,7 +177,8 @@
       if (len >= -112) {
           return len;
       }
-      len = (len < -120) ? -(len + 120) : -(len + 112);
+      boolean isNegative = (len < -120);
+      len = isNegative ? -(len + 120) : -(len + 112);
       if (start+1+len>bytes.length)
           throw new IOException(
                   "Not enough number of bytes for a zero-compressed integer");
@@ -186,7 +187,7 @@
           i = i << 8;
           i = i | (bytes[start+1+idx] & 0xFF);
       }
-      return i;
+      return (isNegative ? (i | 0x8000000000000000L) : i);
   }
   
   /**

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableUtils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableUtils.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableUtils.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/io/WritableUtils.java Wed Feb 28 19:47:27 2007
@@ -312,15 +312,15 @@
       if (len >= -112) {
           return len;
       }
-      len = (len < -120) ? -(len + 120) : -(len + 112);
-      byte[] barr = new byte[len];
-      stream.readFully(barr);
+      boolean isNegative = (len < -120);
+      len = isNegative ? -(len + 120) : -(len + 112);
       long i = 0;
       for (int idx = 0; idx < len; idx++) {
+          byte b = stream.readByte();
           i = i << 8;
-          i = i | (barr[idx] & 0xFF);
+          i = i | (b & 0xFF);
       }
-      return i;
+      return (isNegative ? (i | 0x8000000000000000L) : i);
   }
 
   /**

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java Wed Feb 28 19:47:27 2007
@@ -22,11 +22,6 @@
 import java.io.IOException;
 import java.io.DataInputStream;
 import java.io.InputStream;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-
-import org.apache.hadoop.io.WritableUtils;
-
 
 /**
  *
@@ -34,7 +29,7 @@
  */
 public class BinaryInputArchive implements InputArchive {
     
-    private DataInput in;
+    final private DataInput in;
     
     static BinaryInputArchive getArchive(InputStream strm) {
         return new BinaryInputArchive(new DataInputStream(strm));
@@ -42,8 +37,8 @@
     
     static private class BinaryIndex implements Index {
         private int nelems;
-        BinaryIndex(int nelems) {
-            this.nelems = nelems;
+        private BinaryIndex(int nelems) {
+          this.nelems = nelems;
         }
         public boolean done() {
             return (nelems <= 0);
@@ -57,61 +52,69 @@
         this.in = in;
     }
     
-    public byte readByte(String tag) throws IOException {
+    public byte readByte(final String tag) throws IOException {
         return in.readByte();
     }
     
-    public boolean readBool(String tag) throws IOException {
+    public boolean readBool(final String tag) throws IOException {
         return in.readBoolean();
     }
     
-    public int readInt(String tag) throws IOException {
-        return WritableUtils.readVInt(in);
+    public int readInt(final String tag) throws IOException {
+        return Utils.readVInt(in);
     }
     
-    public long readLong(String tag) throws IOException {
-        return WritableUtils.readVLong(in);
+    public long readLong(final String tag) throws IOException {
+        return Utils.readVLong(in);
     }
     
-    public float readFloat(String tag) throws IOException {
+    public float readFloat(final String tag) throws IOException {
         return in.readFloat();
     }
     
-    public double readDouble(String tag) throws IOException {
+    public double readDouble(final String tag) throws IOException {
         return in.readDouble();
     }
     
-    public Text readString(String tag) throws IOException {
-        Text text = new Text();
-        text.readFields(in);
-        return text;
+    public String readString(final String tag) throws IOException {
+      final int length = Utils.readVInt(in);
+      final byte[] bytes = new byte[length];
+      in.readFully(bytes);
+      return new String(bytes, "UTF-8");
     }
     
-    public BytesWritable readBuffer(String tag) throws IOException {
-      int len = WritableUtils.readVInt(in);
-      byte[] barr = new byte[len];
+    public Buffer readBuffer(final String tag) throws IOException {
+      final int len = Utils.readVInt(in);
+      final byte[] barr = new byte[len];
       in.readFully(barr);
-      return new BytesWritable(barr);
+      return new Buffer(barr);
     }
     
-    public void readRecord(Record r, String tag) throws IOException {
-        r.deserialize(this, tag);
+    public void readRecord(final Record record, final String tag) throws IOException {
+        record.deserialize(this, tag);
     }
     
-    public void startRecord(String tag) throws IOException {}
+    public void startRecord(final String tag) throws IOException {
+      // no-op
+    }
     
-    public void endRecord(String tag) throws IOException {}
+    public void endRecord(final String tag) throws IOException {
+      // no-op
+    }
     
-    public Index startVector(String tag) throws IOException {
-        return new BinaryIndex(readInt(tag));
+    public Index startVector(final String tag) throws IOException {
+      return new BinaryIndex(readInt(tag));
     }
     
-    public void endVector(String tag) throws IOException {}
+    public void endVector(final String tag) throws IOException {
+      // no-op
+}
     
-    public Index startMap(String tag) throws IOException {
+    public Index startMap(final String tag) throws IOException {
         return new BinaryIndex(readInt(tag));
     }
     
-    public void endMap(String tag) throws IOException {}
-    
+    public void endMap(final String tag) throws IOException {
+      // no-op
+    }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java Wed Feb 28 19:47:27 2007
@@ -24,10 +24,6 @@
 import java.io.DataOutput;
 import java.io.DataOutputStream;
 import java.io.OutputStream;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
-
-import org.apache.hadoop.io.WritableUtils;
 
 /**
  *
@@ -55,11 +51,11 @@
     }
     
     public void writeInt(int i, String tag) throws IOException {
-        WritableUtils.writeVInt(out, i);
+        Utils.writeVInt(out, i);
     }
     
     public void writeLong(long l, String tag) throws IOException {
-        WritableUtils.writeVLong(out, l);
+        Utils.writeVLong(out, l);
     }
     
     public void writeFloat(float f, String tag) throws IOException {
@@ -70,15 +66,18 @@
         out.writeDouble(d);
     }
     
-    public void writeString(Text s, String tag) throws IOException {
-        s.write(out);
+    public void writeString(String s, String tag) throws IOException {
+      byte[] bytes = s.getBytes("UTF-8");
+      int length = bytes.length;
+      Utils.writeVInt(out, length);
+      out.write(bytes);
     }
     
-    public void writeBuffer(BytesWritable buf, String tag)
+    public void writeBuffer(Buffer buf, String tag)
     throws IOException {
       byte[] barr = buf.get();
-      int len = buf.getSize();
-      WritableUtils.writeVInt(out, len);
+      int len = buf.getCount();
+      Utils.writeVInt(out, len);
       out.write(barr, 0, len);
     }
     

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Buffer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Buffer.java?view=auto&rev=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Buffer.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Buffer.java Wed Feb 28 19:47:27 2007
@@ -0,0 +1,244 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.UnsupportedEncodingException;
+
+/**
+ * A byte sequence that is used as a Java native type for buffer.
+ * It is resizable and distinguishes between the count of the seqeunce and
+ * the current capacity.
+ * 
+ * @author Milind Bhandarkar
+ */
+public class Buffer implements Comparable, Cloneable {
+  /** Number of valid bytes in this.bytes. */
+  private int count;
+  /** Backing store for Buffer. */
+  private byte[] bytes = null;
+
+  /**
+   * Create a zero-count sequence.
+   */
+  public Buffer() {
+    this.count = 0;
+  }
+
+  /**
+   * Create a Buffer using the byte array as the initial value.
+   *
+   * @param bytes This array becomes the backing storage for the object.
+   */
+  public Buffer(byte[] bytes) {
+    this.bytes = bytes;
+    this.count = (bytes == null) ? 0 : bytes.length;
+  }
+  
+  /**
+   * Create a Buffer using the byte range as the initial value.
+   *
+   * @param bytes Copy of this array becomes the backing storage for the object.
+   * @param offset offset into byte array
+   * @param length length of data
+   */
+  public Buffer(byte[] bytes, int offset, int length) {
+    copy(bytes, offset, length);
+  }
+  
+  
+  /**
+   * Use the specified bytes array as underlying sequence.
+   *
+   * @param bytes byte sequence
+   */
+  public void set(byte[] bytes) {
+    this.count = (bytes == null) ? 0 : bytes.length;
+    this.bytes = bytes;
+  }
+  
+  /**
+   * Copy the specified byte array to the Buffer. Replaces the current buffer.
+   *
+   * @param bytes byte array to be assigned
+   * @param offset offset into byte array
+   * @param length length of data
+   */
+  public final void copy(byte[] bytes, int offset, int length) {
+    if (this.bytes == null || this.bytes.length < length) {
+      this.bytes = new byte[length];
+    }
+    System.arraycopy(bytes, offset, this.bytes, 0, length);
+    this.count = length;
+  }
+  
+  /**
+   * Get the data from the Buffer.
+   * 
+   * @return The data is only valid between 0 and getCount() - 1.
+   */
+  public byte[] get() {
+    if (bytes == null) {
+      bytes = new byte[0];
+    }
+    return bytes;
+  }
+  
+  /**
+   * Get the current count of the buffer.
+   */
+  public int getCount() {
+    return count;
+  }
+  
+  /**
+   * Get the capacity, which is the maximum count that could handled without
+   * resizing the backing storage.
+   * 
+   * @return The number of bytes
+   */
+  public int getCapacity() {
+    return this.get().length;
+  }
+  
+  /**
+   * Change the capacity of the backing storage.
+   * The data is preserved if newCapacity >= getCount().
+   * @param newCapacity The new capacity in bytes.
+   */
+  public void setCapacity(int newCapacity) {
+    if (newCapacity < 0) {
+     throw new IllegalArgumentException("Invalid capacity argument "+newCapacity); 
+    }
+    if (newCapacity == 0) {
+      this.bytes = null;
+      this.count = 0;
+      return;
+    }
+    if (newCapacity != getCapacity()) {
+      byte[] data = new byte[newCapacity];
+      if (newCapacity < count) {
+        count = newCapacity;
+      }
+      if (count != 0) {
+        System.arraycopy(this.get(), 0, data, 0, count);
+      }
+      bytes = data;
+    }
+  }
+  
+  /**
+   * Reset the buffer to 0 size
+   */
+  public void reset() {
+    setCapacity(0);
+  }
+  
+  /**
+   * Change the capacity of the backing store to be the same as the current 
+   * count of buffer.
+   */
+  public void truncate() {
+    setCapacity(count);
+  }
+  
+  /**
+   * Append specified bytes to the buffer.
+   *
+   * @param bytes byte array to be appended
+   * @param offset offset into byte array
+   * @param length length of data
+
+   */
+  public void append(byte[] bytes, int offset, int length) {
+    setCapacity(count+length);
+    System.arraycopy(bytes, offset, this.get(), count, length);
+    count = count + length;
+  }
+  
+  /**
+   * Append specified bytes to the buffer
+   *
+   * @param bytes byte array to be appended
+   */
+  public void append(byte[] bytes) {
+    append(bytes, 0, bytes.length);
+  }
+  
+  // inherit javadoc
+  public int hashCode() {
+    int hash = 1;
+    byte[] b = this.get();
+    for (int i = 0; i < count; i++)
+      hash = (31 * hash) + (int)b[i];
+    return hash;
+  }
+  
+  /**
+   * Define the sort order of the Buffer.
+   * 
+   * @param other The other buffer
+   * @return Positive if this is bigger than other, 0 if they are equal, and
+   *         negative if this is smaller than other.
+   */
+  public int compareTo(Object other) {
+    Buffer right = ((Buffer) other);
+    byte[] lb = this.get();
+    byte[] rb = right.get();
+    for (int i = 0; i < count && i < right.count; i++) {
+      int a = (lb[i] & 0xff);
+      int b = (rb[i] & 0xff);
+      if (a != b) {
+        return a - b;
+      }
+    }
+    return count - right.count;
+  }
+  
+  // inherit javadoc
+  public boolean equals(Object other) {
+    if (other instanceof Buffer && this != other) {
+      return compareTo(other) == 0;
+    }
+    return (this == other);
+  }
+  
+  // inheric javadoc
+  public String toString() {
+    StringBuffer sb = new StringBuffer(2*count);
+    for(int idx = 0; idx < count; idx++) {
+      sb.append(Integer.toHexString((int)bytes[idx]));
+    }
+    return sb.toString();
+  }
+  
+  /**
+   * Convert the byte buffer to a string an specific character encoding
+   *
+   * @param charsetName Valid Java Character Set Name
+   */
+  public String toString(String charsetName)
+  throws UnsupportedEncodingException {
+    return new String(this.get(), 0, this.getCount(), charsetName);
+  }
+  
+  // inherit javadoc
+  public Object clone() throws CloneNotSupportedException {
+    return new Buffer(this.get().clone(), 0, this.getCount());
+  }
+}

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java Wed Feb 28 19:47:27 2007
@@ -23,8 +23,6 @@
 import java.io.IOException;
 import java.io.PushbackReader;
 import java.io.UnsupportedEncodingException;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -121,13 +119,12 @@
         }
     }
     
-    public Text readString(String tag) throws IOException {
+    public String readString(String tag) throws IOException {
         String sval = readField(tag);
         return Utils.fromCSVString(sval);
-        
     }
     
-    public BytesWritable readBuffer(String tag) throws IOException {
+    public Buffer readBuffer(String tag) throws IOException {
         String sval = readField(tag);
         return Utils.fromCSVBuffer(sval);
     }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java Wed Feb 28 19:47:27 2007
@@ -24,8 +24,6 @@
 import java.io.PrintStream;
 import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -91,13 +89,13 @@
         throwExceptionOnError(tag);
     }
     
-    public void writeString(Text s, String tag) throws IOException {
+    public void writeString(String s, String tag) throws IOException {
         printCommaUnlessFirst();
         stream.print(Utils.toCSVString(s));
         throwExceptionOnError(tag);
     }
     
-    public void writeBuffer(BytesWritable buf, String tag)
+    public void writeBuffer(Buffer buf, String tag)
     throws IOException {
         printCommaUnlessFirst();
         stream.print(Utils.toCSVBuffer(buf));

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java Wed Feb 28 19:47:27 2007
@@ -34,6 +34,6 @@
  * @author Milind Bhandarkar
  */
 public interface Index {
-    public boolean done();
-    public void incr();
+    boolean done();
+    void incr();
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java Wed Feb 28 19:47:27 2007
@@ -19,8 +19,6 @@
 package org.apache.hadoop.record;
 
 import java.io.IOException;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  * Interface that all the Deserializers have to implement.
@@ -34,8 +32,8 @@
     public long readLong(String tag) throws IOException;
     public float readFloat(String tag) throws IOException;
     public double readDouble(String tag) throws IOException;
-    public Text readString(String tag) throws IOException;
-    public BytesWritable readBuffer(String tag) throws IOException;
+    public String readString(String tag) throws IOException;
+    public Buffer readBuffer(String tag) throws IOException;
     public void readRecord(Record r, String tag) throws IOException;
     public void startRecord(String tag) throws IOException;
     public void endRecord(String tag) throws IOException;

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java Wed Feb 28 19:47:27 2007
@@ -21,8 +21,6 @@
 import java.io.IOException;
 import java.util.TreeMap;
 import java.util.ArrayList;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  * Interface that alll the serializers have to implement.
@@ -36,8 +34,8 @@
     public void writeLong(long l, String tag) throws IOException;
     public void writeFloat(float f, String tag) throws IOException;
     public void writeDouble(double d, String tag) throws IOException;
-    public void writeString(Text s, String tag) throws IOException;
-    public void writeBuffer(BytesWritable buf, String tag)
+    public void writeString(String s, String tag) throws IOException;
+    public void writeBuffer(Buffer buf, String tag)
         throws IOException;
     public void writeRecord(Record r, String tag) throws IOException;
     public void startRecord(Record r, String tag) throws IOException;

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java Wed Feb 28 19:47:27 2007
@@ -25,9 +25,9 @@
  * 
  * @author Milind Bhandarkar
  */
-public interface Record {
-    public void serialize(OutputArchive archive, String tag)
+public interface Record extends Cloneable {
+    void serialize(OutputArchive archive, String tag)
         throws IOException;
-    public void deserialize(InputArchive archive, String tag)
+    void deserialize(InputArchive archive, String tag)
         throws IOException;
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java Wed Feb 28 19:47:27 2007
@@ -20,13 +20,9 @@
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.DataInputStream;
-import java.io.UnsupportedEncodingException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.HashMap;
-import javax.xml.parsers.ParserConfigurationException;
-import org.xml.sax.SAXException;
 
 /**
  * Front-end interface to deserializers. Also acts as a factory

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java Wed Feb 28 19:47:27 2007
@@ -21,13 +21,10 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.DataOutputStream;
-import java.io.InputStream;
 import java.io.UnsupportedEncodingException;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.util.HashMap;
-import javax.xml.parsers.ParserConfigurationException;
-import org.xml.sax.SAXException;
 
 /**
  * Front-end for serializers. Also serves as a factory for serializers.

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java Wed Feb 28 19:47:27 2007
@@ -18,224 +18,316 @@
 
 package org.apache.hadoop.record;
 
-import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
 import java.io.IOException;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.io.WritableUtils;
 
 /**
  * Various utility functions for Hadooop record I/O runtime.
  * @author Milind Bhandarkar
  */
 public class Utils {
-    
-    /** Cannot create a new instance of Utils */
-    private Utils() {
+  
+  /** Cannot create a new instance of Utils */
+  private Utils() {
+  }
+  
+  public static final char[] hexchars = { '0', '1', '2', '3', '4', '5',
+  '6', '7', '8', '9', 'A', 'B',
+  'C', 'D', 'E', 'F' };
+  /**
+   *
+   * @param s
+   * @return
+   */
+  static String toXMLString(String s) {
+    StringBuffer sb = new StringBuffer();
+    for (int idx = 0; idx < s.length(); idx++) {
+      char ch = s.charAt(idx);
+      if (ch == '<') {
+        sb.append("&lt;");
+      } else if (ch == '&') {
+        sb.append("&amp;");
+      } else if (ch == '%') {
+        sb.append("%25");
+      } else if (ch < 0x20) {
+        sb.append("%");
+        sb.append(hexchars[ch/16]);
+        sb.append(hexchars[ch%16]);
+      } else {
+        sb.append(ch);
+      }
     }
-   
-    public static final char[] hexchars = { '0', '1', '2', '3', '4', '5',
-                                            '6', '7', '8', '9', 'A', 'B',
-                                            'C', 'D', 'E', 'F' };
-    /**
-     * 
-     * @param s 
-     * @return 
-     */
-    static String toXMLString(Text t) {
-        String s = t.toString();
-        StringBuffer sb = new StringBuffer();
-        for (int idx = 0; idx < s.length(); idx++) {
-          char ch = s.charAt(idx);
-          if (ch == '<') {
-            sb.append("&lt;");
-          } else if (ch == '&') {
-            sb.append("&amp;");
-          } else if (ch == '%') {
-            sb.append("%25");
-          } else if (ch < 0x20) {
-            sb.append("%");
-            sb.append(hexchars[ch/16]);
-            sb.append(hexchars[ch%16]);
-          } else {
-            sb.append(ch);
-          }
-        }
-        return sb.toString();
+    return sb.toString();
+  }
+  
+  static private int h2c(char ch) {
+    if (ch >= '0' && ch <= '9') {
+      return ch - '0';
+    } else if (ch >= 'A' && ch <= 'F') {
+      return ch - 'A';
+    } else if (ch >= 'a' && ch <= 'f') {
+      return ch - 'a';
     }
-    
-    static private int h2c(char ch) {
-      if (ch >= '0' && ch <= '9') {
-        return ch - '0';
-      } else if (ch >= 'A' && ch <= 'F') {
-        return ch - 'A';
-      } else if (ch >= 'a' && ch <= 'f') {
-        return ch - 'a';
+    return 0;
+  }
+  
+  /**
+   *
+   * @param s
+   * @return
+   */
+  static String fromXMLString(String s) {
+    StringBuffer sb = new StringBuffer();
+    for (int idx = 0; idx < s.length();) {
+      char ch = s.charAt(idx++);
+      if (ch == '%') {
+        char ch1 = s.charAt(idx++);
+        char ch2 = s.charAt(idx++);
+        char res = (char)(h2c(ch1)*16 + h2c(ch2));
+        sb.append(res);
+      } else {
+        sb.append(ch);
       }
-      return 0;
     }
     
-    /**
-     * 
-     * @param s 
-     * @return 
-     */
-    static Text fromXMLString(String s) {
-        StringBuffer sb = new StringBuffer();
-        for (int idx = 0; idx < s.length();) {
-          char ch = s.charAt(idx++);
-          if (ch == '%') {
-            char ch1 = s.charAt(idx++);
-            char ch2 = s.charAt(idx++);
-            char res = (char)(h2c(ch1)*16 + h2c(ch2));
-            sb.append(res);
-          } else {
-            sb.append(ch);
-          }
-        }
-        
-        return new Text(sb.toString());
+    return sb.toString();
+  }
+  
+  /**
+   *
+   * @param s
+   * @return
+   */
+  static String toCSVString(String s) {
+    StringBuffer sb = new StringBuffer(s.length()+1);
+    sb.append('\'');
+    int len = s.length();
+    for (int i = 0; i < len; i++) {
+      char c = s.charAt(i);
+      switch(c) {
+        case '\0':
+          sb.append("%00");
+          break;
+        case '\n':
+          sb.append("%0A");
+          break;
+        case '\r':
+          sb.append("%0D");
+          break;
+        case ',':
+          sb.append("%2C");
+          break;
+        case '}':
+          sb.append("%7D");
+          break;
+        case '%':
+          sb.append("%25");
+          break;
+        default:
+          sb.append(c);
+      }
     }
-    
-    /**
-     * 
-     * @param s 
-     * @return 
-     */
-    static String toCSVString(Text t) {
-        String s = t.toString();
-        StringBuffer sb = new StringBuffer(s.length()+1);
-        sb.append('\'');
-        int len = s.length();
-        for (int i = 0; i < len; i++) {
-            char c = s.charAt(i);
-            switch(c) {
-                case '\0':
-                    sb.append("%00");
-                    break;
-                case '\n':
-                    sb.append("%0A");
-                    break;
-                case '\r':
-                    sb.append("%0D");
-                    break;
-                case ',':
-                    sb.append("%2C");
-                    break;
-                case '}':
-                    sb.append("%7D");
-                    break;
-                case '%':
-                    sb.append("%25");
-                    break;
-                default:
-                    sb.append(c);
-            }
-        }
-        return sb.toString();
+    return sb.toString();
+  }
+  
+  /**
+   *
+   * @param s
+   * @throws java.io.IOException
+   * @return
+   */
+  static String fromCSVString(String s) throws IOException {
+    if (s.charAt(0) != '\'') {
+      throw new IOException("Error deserializing string.");
     }
-    
-    /**
-     * 
-     * @param s 
-     * @throws java.io.IOException 
-     * @return 
-     */
-    static Text fromCSVString(String s) throws IOException {
-        if (s.charAt(0) != '\'') {
-            throw new IOException("Error deserializing string.");
+    int len = s.length();
+    StringBuffer sb = new StringBuffer(len-1);
+    for (int i = 1; i < len; i++) {
+      char c = s.charAt(i);
+      if (c == '%') {
+        char ch1 = s.charAt(i+1);
+        char ch2 = s.charAt(i+2);
+        i += 2;
+        if (ch1 == '0' && ch2 == '0') {
+          sb.append('\0');
+        } else if (ch1 == '0' && ch2 == 'A') {
+          sb.append('\n');
+        } else if (ch1 == '0' && ch2 == 'D') {
+          sb.append('\r');
+        } else if (ch1 == '2' && ch2 == 'C') {
+          sb.append(',');
+        } else if (ch1 == '7' && ch2 == 'D') {
+          sb.append('}');
+        } else if (ch1 == '2' && ch2 == '5') {
+          sb.append('%');
+        } else {
+          throw new IOException("Error deserializing string.");
         }
-        int len = s.length();
-        StringBuffer sb = new StringBuffer(len-1);
-        for (int i = 1; i < len; i++) {
-            char c = s.charAt(i);
-            if (c == '%') {
-                char ch1 = s.charAt(i+1);
-                char ch2 = s.charAt(i+2);
-                i += 2;
-                if (ch1 == '0' && ch2 == '0') { sb.append('\0'); }
-                else if (ch1 == '0' && ch2 == 'A') { sb.append('\n'); }
-                else if (ch1 == '0' && ch2 == 'D') { sb.append('\r'); }
-                else if (ch1 == '2' && ch2 == 'C') { sb.append(','); }
-                else if (ch1 == '7' && ch2 == 'D') { sb.append('}'); }
-                else if (ch1 == '2' && ch2 == '5') { sb.append('%'); }
-                else {throw new IOException("Error deserializing string.");}
-            } else {
-                sb.append(c);
-            }
-        }
-        return new Text(sb.toString());
-    }
-    
-    /**
-     * 
-     * @param s 
-     * @return 
-     */
-    static String toXMLBuffer(BytesWritable s) {
-        byte[] barr = s.get();
-        int bsize = s.getSize();
-        StringBuffer sb = new StringBuffer(2*bsize);
-        for (int idx = 0; idx < bsize; idx++) {
-            sb.append(Integer.toHexString((int)barr[idx]));
-        }
-        return sb.toString();
+      } else {
+        sb.append(c);
+      }
     }
-    
-    /**
-     * 
-     * @param s 
-     * @throws java.io.IOException 
-     * @return 
-     */
-    static BytesWritable fromXMLBuffer(String s)
-    throws IOException {
-        if (s.length() == 0) { return new BytesWritable(); }
-        int blen = s.length()/2;
-        byte[] barr = new byte[blen];
-        for (int idx = 0; idx < blen; idx++) {
-            char c1 = s.charAt(2*idx);
-            char c2 = s.charAt(2*idx+1);
-            barr[idx] = Byte.parseByte(""+c1+c2, 16);
-        }
-        return new BytesWritable(barr);
+    return sb.toString();
+  }
+  
+  /**
+   *
+   * @param s
+   * @return
+   */
+  static String toXMLBuffer(Buffer s) {
+    return s.toString();
+  }
+  
+  /**
+   *
+   * @param s
+   * @throws java.io.IOException
+   * @return
+   */
+  static Buffer fromXMLBuffer(String s)
+  throws IOException {
+    if (s.length() == 0) { return new Buffer(); }
+    int blen = s.length()/2;
+    byte[] barr = new byte[blen];
+    for (int idx = 0; idx < blen; idx++) {
+      char c1 = s.charAt(2*idx);
+      char c2 = s.charAt(2*idx+1);
+      barr[idx] = Byte.parseByte(""+c1+c2, 16);
     }
-    
-    /**
-     * 
-     * @param buf 
-     * @return 
-     */
-    static String toCSVBuffer(BytesWritable buf) {
-        byte[] barr = buf.get();
-        int bsize = buf.getSize();
-        StringBuffer sb = new StringBuffer(bsize+1);
-        sb.append('#');
-        for(int idx = 0; idx < bsize; idx++) {
-            sb.append(Integer.toHexString((int)barr[idx]));
-        }
-        return sb.toString();
+    return new Buffer(barr);
+  }
+  
+  /**
+   *
+   * @param buf
+   * @return
+   */
+  static String toCSVBuffer(Buffer buf) {
+    StringBuffer sb = new StringBuffer("#");
+    sb.append(buf.toString());
+    return sb.toString();
+  }
+  
+  /**
+   * Converts a CSV-serialized representation of buffer to a new
+   * Buffer
+   * @param s CSV-serialized representation of buffer
+   * @throws java.io.IOException
+   * @return Deserialized Buffer
+   */
+  static Buffer fromCSVBuffer(String s)
+  throws IOException {
+    if (s.charAt(0) != '#') {
+      throw new IOException("Error deserializing buffer.");
     }
-    
-    /**
-     * Converts a CSV-serialized representation of buffer to a new
-     * BytesWritable.
-     * @param s CSV-serialized representation of buffer
-     * @throws java.io.IOException 
-     * @return Deserialized BytesWritable
-     */
-    static BytesWritable fromCSVBuffer(String s)
-    throws IOException {
-        if (s.charAt(0) != '#') {
-            throw new IOException("Error deserializing buffer.");
-        }
-        if (s.length() == 1) { return new BytesWritable(); }
-        int blen = (s.length()-1)/2;
-        byte[] barr = new byte[blen];
-        for (int idx = 0; idx < blen; idx++) {
-            char c1 = s.charAt(2*idx+1);
-            char c2 = s.charAt(2*idx+2);
-            barr[idx] = Byte.parseByte(""+c1+c2, 16);
-        }
-        return new BytesWritable(barr);
+    if (s.length() == 1) { return new Buffer(); }
+    int blen = (s.length()-1)/2;
+    byte[] barr = new byte[blen];
+    for (int idx = 0; idx < blen; idx++) {
+      char c1 = s.charAt(2*idx+1);
+      char c2 = s.charAt(2*idx+2);
+      barr[idx] = Byte.parseByte(""+c1+c2, 16);
     }
+    return new Buffer(barr);
+  }
+  
+  /** Parse a float from a byte array. */
+  public static float readFloat(byte[] bytes, int start) {
+    return WritableComparator.readFloat(bytes, start);
+  }
+  
+  /** Parse a double from a byte array. */
+  public static double readDouble(byte[] bytes, int start) {
+    return WritableComparator.readDouble(bytes, start);
+  }
+  
+  /**
+   * Reads a zero-compressed encoded long from a byte array and returns it.
+   * @param bytes byte array with decode long
+   * @param start starting index
+   * @throws java.io.IOException
+   * @return deserialized long
+   */
+  public static long readVLong(byte[] bytes, int start) throws IOException {
+    return WritableComparator.readVLong(bytes, start);
+  }
+  
+  /**
+   * Reads a zero-compressed encoded integer from a byte array and returns it.
+   * @param bytes byte array with the encoded integer
+   * @param start start index
+   * @throws java.io.IOException
+   * @return deserialized integer
+   */
+  public static int readVInt(byte[] bytes, int start) throws IOException {
+    return WritableComparator.readVInt(bytes, start);
+  }
+  
+  /**
+   * Reads a zero-compressed encoded long from a stream and return it.
+   * @param in input stream
+   * @throws java.io.IOException
+   * @return deserialized long
+   */
+  public static long readVLong(DataInput in) throws IOException {
+    return WritableUtils.readVLong(in);
+  }
+  
+  /**
+   * Reads a zero-compressed encoded integer from a stream and returns it.
+   * @param in input stream
+   * @throws java.io.IOException
+   * @return deserialized integer
+   */
+  public static int readVInt(DataInput in) throws IOException {
+    return WritableUtils.readVInt(in);
+  }
+  
+  /**
+   * Get the encoded length if an integer is stored in a variable-length format
+   * @return the encoded length
+   */
+  public static int getVIntSize(long i) {
+    return WritableUtils.getVIntSize(i);
+  }
+  
+  /**
+   * Serializes a long to a binary stream with zero-compressed encoding.
+   * For -112 <= i <= 127, only one byte is used with the actual value.
+   * For other values of i, the first byte value indicates whether the
+   * long is positive or negative, and the number of bytes that follow.
+   * If the first byte value v is between -113 and -120, the following long
+   * is positive, with number of bytes that follow are -(v+112).
+   * If the first byte value v is between -121 and -128, the following long
+   * is negative, with number of bytes that follow are -(v+120). Bytes are
+   * stored in the high-non-zero-byte-first order.
+   *
+   * @param stream Binary output stream
+   * @param i Long to be serialized
+   * @throws java.io.IOException
+   */
+  public static void writeVLong(DataOutput stream, long i) throws IOException {
+    WritableUtils.writeVLong(stream, i);
+  }
+  
+  /**
+   * Serializes an int to a binary stream with zero-compressed encoding.
+   *
+   * @param stream Binary output stream
+   * @param i int to be serialized
+   * @throws java.io.IOException
+   */
+  public static void writeVInt(DataOutput stream, int i) throws IOException {
+    WritableUtils.writeVInt(stream, i);
+  }
+  
+  /** Lexicographic order of binary data. */
+  public static int compareBytes(byte[] b1, int s1, int l1,
+      byte[] b2, int s2, int l2) {
+    return WritableComparator.compareBytes(b1, s1, l1, b2, s2, l2);
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java Wed Feb 28 19:47:27 2007
@@ -27,8 +27,7 @@
 import javax.xml.parsers.SAXParserFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.parsers.SAXParser;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
+
 /**
  *
  * @author Milind Bhandarkar
@@ -199,7 +198,7 @@
         return Double.parseDouble(v.getValue());
     }
     
-    public Text readString(String tag) throws IOException {
+    public String readString(String tag) throws IOException {
         Value v = next();
         if (!"string".equals(v.getType())) {
             throw new IOException("Error deserializing "+tag+".");
@@ -207,7 +206,7 @@
         return Utils.fromXMLString(v.getValue());
     }
     
-    public BytesWritable readBuffer(String tag) throws IOException {
+    public Buffer readBuffer(String tag) throws IOException {
         Value v = next();
         if (!"string".equals(v.getType())) {
             throw new IOException("Error deserializing "+tag+".");

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java Wed Feb 28 19:47:27 2007
@@ -24,8 +24,6 @@
 import java.io.PrintStream;
 import java.io.OutputStream;
 import java.util.Stack;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.Text;
 
 /**
  *
@@ -191,7 +189,7 @@
         printEndEnvelope(tag);
     }
     
-    public void writeString(Text s, String tag) throws IOException {
+    public void writeString(String s, String tag) throws IOException {
         printBeginEnvelope(tag);
         stream.print("<string>");
         stream.print(Utils.toXMLString(s));
@@ -199,7 +197,7 @@
         printEndEnvelope(tag);
     }
     
-    public void writeBuffer(BytesWritable buf, String tag)
+    public void writeBuffer(Buffer buf, String tag)
     throws IOException {
         printBeginEnvelope(tag);
         stream.print("<string>");

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CGenerator.java?view=auto&rev=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CGenerator.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CGenerator.java Wed Feb 28 19:47:27 2007
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+import java.util.ArrayList;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * C Code generator front-end for Hadoop record I/O.
+ *
+ * @author Milind Bhandarkar
+ */
+class CGenerator extends CodeGenerator {
+  
+  CGenerator() {
+  }
+  
+  /**
+   * Generate C code. This method only creates the requested file(s)
+   * and spits-out file-level elements (such as include statements etc.)
+   * record-level code is generated by JRecord.
+   */
+  void genCode(String name, ArrayList<JFile> ilist,
+      ArrayList<JRecord> rlist, String destDir, ArrayList<String> options)
+      throws IOException {
+    name = new File(destDir, (new File(name)).getName()).getAbsolutePath();
+    FileWriter cc = new FileWriter(name+".c");
+    FileWriter hh = new FileWriter(name+".h");
+    
+    hh.write("#ifndef __"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#define __"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#include \"recordio.h\"\n");
+    for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
+      hh.write("#include \""+iter.next().getName()+".h\"\n");
+    }
+    
+    cc.write("#include \""+name+".h\"\n");
+    
+    /*
+    for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
+      iter.next().genCppCode(hh, cc);
+    }
+    */
+    
+    hh.write("#endif //"+name.toUpperCase().replace('.','_')+"__\n");
+    
+    hh.close();
+    cc.close();
+  }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeBuffer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeBuffer.java?view=auto&rev=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeBuffer.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeBuffer.java Wed Feb 28 19:47:27 2007
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.record.compiler;
+
+import java.util.ArrayList;
+
+/**
+ * A wrapper around StringBuffer that automatically does indentation
+ *
+ * @author Milind Bhandarkar
+ */
+public class CodeBuffer {
+  
+  static private ArrayList<Character> startMarkers = new ArrayList<Character>();
+  static private ArrayList<Character> endMarkers = new ArrayList<Character>();
+  
+  static {
+    addMarkers('{', '}');
+    addMarkers('(', ')');
+  }
+  
+  static void addMarkers(char ch1, char ch2) {
+    startMarkers.add(ch1);
+    endMarkers.add(ch2);
+  }
+  
+  private int level = 0;
+  private int numSpaces = 2;
+  private boolean firstChar = true;
+  private StringBuffer sb;
+  
+  /** Creates a new instance of CodeBuffer */
+  CodeBuffer() {
+    this(2, "");
+  }
+  
+  CodeBuffer(String s) {
+    this(2, s);
+  }
+  
+  CodeBuffer(int numSpaces, String s) {
+    sb = new StringBuffer();
+    this.numSpaces = numSpaces;
+    this.append(s);
+  }
+  
+  void append(String s) {
+    int length = s.length();
+    for (int idx = 0; idx < length; idx++) {
+      char ch = s.charAt(idx);
+      append(ch);
+    }
+  }
+  
+  void append(char ch) {
+    if (endMarkers.contains(ch)) {
+      level--;
+    }
+    if (firstChar) {
+      for (int idx = 0; idx < level; idx++) {
+        for (int num = 0; num < numSpaces; num++) {
+          rawAppend(' ');
+        }
+      }
+    }
+    rawAppend(ch);
+    firstChar = false;
+    if (startMarkers.contains(ch)) {
+      level++;
+    }
+    if (ch == '\n') {
+      firstChar = true;
+    }
+  }
+
+  private void rawAppend(char ch) {
+    sb.append(ch);
+  }
+  
+  public String toString() {
+    return sb.toString();
+  }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeGenerator.java?view=auto&rev=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeGenerator.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CodeGenerator.java Wed Feb 28 19:47:27 2007
@@ -0,0 +1,52 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.record.compiler;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/**
+ * CodeGenerator is a Factory and a base class for Hadoop Record I/O translators.
+ * Different translators register creation methods with this factory.
+ *
+ * @author Milind Bhandarkar
+ */
+abstract class CodeGenerator {
+  
+  private static HashMap<String, CodeGenerator> generators =
+      new HashMap<String, CodeGenerator>();
+  
+  static {
+    register("c", new CGenerator());
+    register("c++", new CppGenerator());
+    register("java", new JavaGenerator());
+  }
+  
+  static void register(String lang, CodeGenerator gen) {
+    generators.put(lang, gen);
+  }
+  
+  static CodeGenerator get(String lang) {
+    return generators.get(lang);
+  }
+  
+  abstract void genCode(String file,
+      ArrayList<JFile> inclFiles,
+      ArrayList<JRecord> records,
+      String destDir,
+      ArrayList<String> options) throws IOException;
+}

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java Wed Feb 28 19:47:27 2007
@@ -29,50 +29,39 @@
  *
  * @author Milind Bhandarkar
  */
-class CppGenerator {
-    private String mName;
-    private ArrayList mInclFiles;
-    private ArrayList mRecList;
+class CppGenerator extends CodeGenerator {
+  
+  CppGenerator() {
+  }
+  
+  /**
+   * Generate C++ code. This method only creates the requested file(s)
+   * and spits-out file-level elements (such as include statements etc.)
+   * record-level code is generated by JRecord.
+   */
+  void genCode(String name, ArrayList<JFile> ilist,
+      ArrayList<JRecord> rlist, String destDir, ArrayList<String> options)
+      throws IOException {
+    name = new File(destDir, (new File(name)).getName()).getAbsolutePath();
+    FileWriter cc = new FileWriter(name+".cc");
+    FileWriter hh = new FileWriter(name+".hh");
     
-    /** Creates a new instance of CppGenerator
-     *
-     * @param name possibly full pathname to the file
-     * @param ilist included files (as JFile)
-     * @param rlist List of records defined within this file
-     * @param destDir Output directory
-     */
-    CppGenerator(String name, ArrayList ilist, ArrayList rlist, String destDir) {
-        mName = new File(destDir, (new File(name)).getName()).getAbsolutePath();
-        mInclFiles = ilist;
-        mRecList = rlist;
+    hh.write("#ifndef __"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#define __"+name.toUpperCase().replace('.','_')+"__\n");
+    hh.write("#include \"recordio.hh\"\n");
+    for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
+      hh.write("#include \""+iter.next().getName()+".hh\"\n");
     }
     
-    /**
-     * Generate C++ code. This method only creates the requested file(s)
-     * and spits-out file-level elements (such as include statements etc.)
-     * record-level code is generated by JRecord.
-     */
-    void genCode() throws IOException {
-        FileWriter cc = new FileWriter(mName+".cc");
-        FileWriter hh = new FileWriter(mName+".hh");
-        hh.write("#ifndef __"+mName.toUpperCase().replace('.','_')+"__\n");
-        hh.write("#define __"+mName.toUpperCase().replace('.','_')+"__\n");
-        
-        hh.write("#include \"recordio.hh\"\n");
-        for (Iterator i = mInclFiles.iterator(); i.hasNext();) {
-            JFile f = (JFile) i.next();
-            hh.write("#include \""+f.getName()+".hh\"\n");
-        }
-        cc.write("#include \""+mName+".hh\"\n");
-        
-        for (Iterator i = mRecList.iterator(); i.hasNext();) {
-            JRecord jr = (JRecord) i.next();
-            jr.genCppCode(hh, cc);
-        }
-        
-        hh.write("#endif //"+mName.toUpperCase().replace('.','_')+"__\n");
-        
-        hh.close();
-        cc.close();
+    cc.write("#include \""+name+".hh\"\n");
+    
+    for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
+      iter.next().genCppCode(hh, cc, options);
     }
+    
+    hh.write("#endif //"+name.toUpperCase().replace('.','_')+"__\n");
+    
+    hh.close();
+    cc.close();
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java Wed Feb 28 19:47:27 2007
@@ -18,63 +18,63 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
  *
  * @author Milind Bhandarkar
  */
 public class JBoolean extends JType {
+  
+  class JavaBoolean extends JType.JavaType {
     
-    /** Creates a new instance of JBoolean */
-    public JBoolean() {
-        super("bool", "boolean", "Bool", "Boolean", "toBoolean");
+    JavaBoolean() {
+      super("boolean", "Bool", "Boolean");
     }
     
-    public String getSignature() {
-        return "z";
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      cb.append("ret = ("+fname+" == "+other+")? 0 : ("+fname+"?1:-1);\n");
     }
     
-    String getJavaCompareToWrapper(String fname, String other) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("          boolean ee1 = ("+fname+".toBoolean()\n");
-      sb.append("          boolean ee2 = ("+other+".toBoolean()\n");
-      sb.append("          ret = (ee1 == ee2)? 0 :(ee1?-1:1);\n");
-      sb.append("        }\n");
-      return sb.toString();
-    }
-    
-    public String genJavaCompareTo(String fname, String other) {
-        return "    ret = ("+fname+" == "+other+")? 0 : ("+fname+"?1:-1);\n";
-    }
-    
-    public String genJavaHashCode(String fname) {
-        return "     ret = ("+fname+")?0:1;\n";
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = ("+fname+")?0:1;\n");
     }
     
     // In Binary format, boolean is written as byte. true = 1, false = 0
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if ("+l+"<1) {\n");
-      sb.append("             throw new IOException(\"Boolean is exactly 1 byte. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           "+s+"++; "+l+"--;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("if ("+l+"<1) {\n");
+      cb.append("throw new java.io.IOException(\"Boolean is exactly 1 byte."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append(s+"++; "+l+"--;\n");
+      cb.append("}\n");
     }
     
     // In Binary format, boolean is written as byte. true = 1, false = 0
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if (l1<1 || l2<1) {\n");
-      sb.append("             throw new IOException(\"Boolean is exactly 1 byte. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           if (b1[s1] != b2[s2]) {\n");
-      sb.append("             return (b1[s1]<b2[s2])? -1 : 0;\n");
-      sb.append("           }\n");
-      sb.append("           s1++; s2++; l1--; l2--;\n");
-      sb.append("        }\n");
-      return sb.toString();
-    }
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("if (l1<1 || l2<1) {\n");
+      cb.append("throw new java.io.IOException(\"Boolean is exactly 1 byte."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append("if (b1[s1] != b2[s2]) {\n");
+      cb.append("return (b1[s1]<b2[s2])? -1 : 0;\n");
+      cb.append("}\n");
+      cb.append("s1++; s2++; l1--; l2--;\n");
+      cb.append("}\n");
+    }
+  }
+  
+  /** Creates a new instance of JBoolean */
+  public JBoolean() {
+    setJavaType(new JavaBoolean());
+    setCppType(new CppType("bool"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "z";
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java Wed Feb 28 19:47:27 2007
@@ -18,88 +18,83 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JCompType.CCompType;
+
 /**
+ * Code generator for "buffer" type.
  *
  * @author Milind Bhandarkar
  */
 public class JBuffer extends JCompType {
+  
+  class JavaBuffer extends JavaCompType {
     
-    /** Creates a new instance of JBuffer */
-    public JBuffer() {
-        super(" ::std::string", "BytesWritable", "Buffer", "BytesWritable");
+    JavaBuffer() {
+      super("org.apache.hadoop.record.Buffer", "Buffer", "org.apache.hadoop.record.Buffer");
     }
     
-    public String genCppGetSet(String fname, int fIdx) {
-        String cgetFunc = "  virtual const "+getCppType()+"& get"+fname+"() const {\n";
-        cgetFunc += "    return m"+fname+";\n";
-        cgetFunc += "  }\n";
-        String getFunc = "  virtual "+getCppType()+"& get"+fname+"() {\n";
-        getFunc += "    return m"+fname+";\n";
-        getFunc += "  }\n";
-        return cgetFunc + getFunc;
+    void genWriteMethod(CodeBuffer cb, String fname, String tag) {
+      cb.append("a_.writeBuffer("+fname+",\""+tag+"\");\n");
     }
     
-    public String getSignature() {
-        return "B";
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      cb.append("ret = "+fname+".compareTo("+other+");\n");
     }
     
-    public String genJavaReadWrapper(String fname, String tag, boolean decl) {
-        String ret = "";
-        if (decl) {
-            ret = "    BytesWritable "+fname+";\n";
-        }
-        return ret + "        "+fname+"=a_.readBuffer(\""+tag+"\");\n";
+    void genEquals(CodeBuffer cb, String fname, String peer) {
+      cb.append("ret = "+fname+".equals("+peer+");\n");
     }
     
-    public String genJavaWriteWrapper(String fname, String tag) {
-        return "        a_.writeBuffer("+fname+",\""+tag+"\");\n";
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = "+fname+".hashCode();\n");
     }
     
-    public String genJavaCompareTo(String fname, String other) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("    {\n");
-      sb.append("      byte[] my = "+fname+".get();\n");
-      sb.append("      byte[] ur = "+other+".get();\n");
-      sb.append("      ret = WritableComparator.compareBytes(my,0,"+
-          fname+".getSize(),ur,0,"+other+".getSize());\n");
-      sb.append("    }\n");
-      return sb.toString();
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("int i = org.apache.hadoop.record.Utils.readVInt("+
+          b+", "+s+");\n");
+      cb.append("int z = org.apache.hadoop.record.Utils.getVIntSize(i);\n");
+      cb.append(s+" += z+i; "+l+" -= (z+i);\n");
+      cb.append("}\n");
     }
     
-    public String genJavaCompareToWrapper(String fname, String other) {
-      return "    "+genJavaCompareTo(fname, other);
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("int i1 = org.apache.hadoop.record.Utils.readVInt(b1, s1);\n");
+      cb.append("int i2 = org.apache.hadoop.record.Utils.readVInt(b2, s2);\n");
+      cb.append("int z1 = org.apache.hadoop.record.Utils.getVIntSize(i1);\n");
+      cb.append("int z2 = org.apache.hadoop.record.Utils.getVIntSize(i2);\n");
+      cb.append("s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
+      cb.append("int r1 = org.apache.hadoop.record.Utils.compareBytes(b1,s1,l1,b2,s2,l2);\n");
+      cb.append("if (r1 != 0) { return (r1<0)?-1:0; }\n");
+      cb.append("s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
+      cb.append("}\n");
     }
+  }
+  
+  class CppBuffer extends CppCompType {
     
-    public String genJavaEquals(String fname, String peer) {
-        return "    ret = "+fname+".equals("+peer+");\n";
+    CppBuffer() {
+      super(" ::std::string");
     }
     
-    public String genJavaHashCode(String fname) {
-        return "    ret = "+fname+".hashCode();\n";
-    }
-    
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int i = WritableComparator.readVInt("+b+", "+s+");\n");
-      sb.append("           int z = WritableUtils.getVIntSize(i);\n");
-      sb.append("           "+s+" += z+i; "+l+" -= (z+i);\n");
-      sb.append("        }\n");
-      return sb.toString();
-    }
-    
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           int i1 = WritableComparator.readVInt(b1, s1);\n");
-      sb.append("           int i2 = WritableComparator.readVInt(b2, s2);\n");
-      sb.append("           int z1 = WritableUtils.getVIntSize(i1);\n");
-      sb.append("           int z2 = WritableUtils.getVIntSize(i2);\n");
-      sb.append("           s1+=z1; s2+=z2; l1-=z1; l2-=z2;\n");
-      sb.append("           int r1 = WritableComparator.compareBytes(b1,s1,l1,b2,s2,l2);\n");
-      sb.append("           if (r1 != 0) { return (r1<0)?-1:0; }\n");
-      sb.append("           s1+=i1; s2+=i2; l1-=i1; l1-=i2;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genGetSet(CodeBuffer cb, String fname) {
+      cb.append("virtual const "+getType()+"& get"+toCamelCase(fname)+"() const {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
+      cb.append("virtual "+getType()+"& get"+toCamelCase(fname)+"() {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
     }
+  }
+  /** Creates a new instance of JBuffer */
+  public JBuffer() {
+    setJavaType(new JavaBuffer());
+    setCppType(new CppBuffer());
+    setCType(new CCompType());
+  }
+  
+  String getSignature() {
+    return "B";
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java Wed Feb 28 19:47:27 2007
@@ -18,43 +18,52 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
- *
+ * Code generator for "byte" type.
  * @author Milind Bhandarkar
  */
 public class JByte extends JType {
+  
+  class JavaByte extends JavaType {
     
-    /** Creates a new instance of JByte */
-    public JByte() {
-        super("int8_t", "byte", "Byte", "Byte", "toByte");
+    JavaByte() {
+      super("byte", "Byte", "Byte");
     }
     
-    public String getSignature() {
-        return "b";
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("if ("+l+"<1) {\n");
+      cb.append("throw new java.io.IOException(\"Byte is exactly 1 byte."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append(s+"++; "+l+"--;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if ("+l+"<1) {\n");
-      sb.append("             throw new IOException(\"Byte is exactly 1 byte. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           "+s+"++; "+l+"--;\n");
-      sb.append("        }\n");
-      return sb.toString();
-    }
-    
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if (l1<1 || l2<1) {\n");
-      sb.append("             throw new IOException(\"Byte is exactly 1 byte. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           if (b1[s1] != b2[s2]) {\n");
-      sb.append("             return (b1[s1]<b2[s2])?-1:0;\n");
-      sb.append("           }\n");
-      sb.append("           s1++; s2++; l1--; l2--;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("if (l1<1 || l2<1) {\n");
+      cb.append("throw new java.io.IOException(\"Byte is exactly 1 byte."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append("if (b1[s1] != b2[s2]) {\n");
+      cb.append("return (b1[s1]<b2[s2])?-1:0;\n");
+      cb.append("}\n");
+      cb.append("s1++; s2++; l1--; l2--;\n");
+      cb.append("}\n");
     }
+  }
+  
+  public JByte() {
+    setJavaType(new JavaByte());
+    setCppType(new CppType("int8_t"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "b";
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java Wed Feb 28 19:47:27 2007
@@ -25,35 +25,47 @@
  * @author Milind Bhandarkar
  */
 abstract class JCompType extends JType {
+  
+  abstract class JavaCompType extends JavaType {
     
-    /** Creates a new instance of JCompType */
-    JCompType(String cppType, String javaType, String suffix, String wrapper) {
-        super(cppType, javaType, suffix, wrapper, null);
+    JavaCompType(String type, String suffix, String wrapper) {
+      super(type, suffix, wrapper);
     }
     
-    String genCppGetSet(String fname, int fIdx) {
-        String cgetFunc = "  virtual const "+getCppType()+"& get"+fname+"() const {\n";
-        cgetFunc += "    return m"+fname+";\n";
-        cgetFunc += "  }\n";
-        String getFunc = "  virtual "+getCppType()+"& get"+fname+"() {\n";
-        getFunc += "    return m"+fname+";\n";
-        getFunc += "  }\n";
-        return cgetFunc + getFunc;
+    void genCompareTo(CodeBuffer cb, String fname, String other) {
+      cb.append("ret = "+fname+".compareTo("+other+");\n");
     }
     
-    String genJavaCompareTo(String fname, String other) {
-        return "    ret = "+fname+".compareTo("+other+");\n";
+    void genEquals(CodeBuffer cb, String fname, String peer) {
+      cb.append("ret = "+fname+".equals("+peer+");\n");
     }
     
-    String genJavaCompareToWrapper(String fname, String other) {
-        return "    "+genJavaCompareTo(fname, other);
+    void genHashCode(CodeBuffer cb, String fname) {
+      cb.append("ret = "+fname+".hashCode();\n");
     }
     
-    String genJavaEquals(String fname, String peer) {
-        return "    ret = "+fname+".equals("+peer+");\n";
+    void genClone(CodeBuffer cb, String fname) {
+      cb.append("other."+fname+" = ("+getType()+") this."+fname+".clone();\n");
     }
+  }
+  
+  class CppCompType extends CppType {
     
-    String genJavaHashCode(String fname) {
-        return "    ret = "+fname+".hashCode();\n";
+    CppCompType(String type) {
+      super(type);
     }
+    
+    void genGetSet(CodeBuffer cb, String fname) {
+      cb.append("virtual const "+getType()+"& get"+toCamelCase(fname)+"() const {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
+      cb.append("virtual "+getType()+"& get"+toCamelCase(fname)+"() {\n");
+      cb.append("return "+fname+";\n");
+      cb.append("}\n");
+    }
+  }
+  
+  class CCompType extends CType {
+    
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java Wed Feb 28 19:47:27 2007
@@ -18,50 +18,59 @@
 
 package org.apache.hadoop.record.compiler;
 
+import org.apache.hadoop.record.compiler.JType.CType;
+import org.apache.hadoop.record.compiler.JType.CppType;
+
 /**
  *
  * @author Milind Bhandarkar
  */
 public class JDouble extends JType {
+  
+  class JavaDouble extends JavaType {
     
-    /** Creates a new instance of JDouble */
-    public JDouble() {
-        super("double", "double", "Double", "Double", "toDouble");
+    JavaDouble() {
+      super("double", "Double", "Double");
     }
     
-    public String getSignature() {
-        return "d";
+    void genHashCode(CodeBuffer cb, String fname) {
+      String tmp = "Double.doubleToLongBits("+fname+")";
+      cb.append("ret = (int)("+tmp+"^("+tmp+">>>32));\n");
     }
     
-    public String genJavaHashCode(String fname) {
-        String tmp = "Double.doubleToLongBits("+fname+")";
-        return "    ret = (int)("+tmp+"^("+tmp+">>>32));\n";
+    void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
+      cb.append("{\n");
+      cb.append("if ("+l+"<8) {\n");
+      cb.append("throw new java.io.IOException(\"Double is exactly 8 bytes."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append(s+"+=8; "+l+"-=8;\n");
+      cb.append("}\n");
     }
     
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if ("+l+"<8) {\n");
-      sb.append("             throw new IOException(\"Double is exactly 8 bytes. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           "+s+"+=8; "+l+"-=8;\n");
-      sb.append("        }\n");
-      return sb.toString();
-    }
-    
-    public String genJavaCompareBytes() {
-      StringBuffer sb = new StringBuffer();
-      sb.append("        {\n");
-      sb.append("           if (l1<8 || l2<8) {\n");
-      sb.append("             throw new IOException(\"Double is exactly 8 bytes. Provided buffer is smaller.\");\n");
-      sb.append("           }\n");
-      sb.append("           double d1 = WritableComparator.readDouble(b1, s1);\n");
-      sb.append("           double d2 = WritableComparator.readDouble(b2, s2);\n");
-      sb.append("           if (d1 != d2) {\n");
-      sb.append("             return ((d1-d2) < 0) ? -1 : 0;\n");
-      sb.append("           }\n");
-      sb.append("           s1+=8; s2+=8; l1-=8; l2-=8;\n");
-      sb.append("        }\n");
-      return sb.toString();
+    void genCompareBytes(CodeBuffer cb) {
+      cb.append("{\n");
+      cb.append("if (l1<8 || l2<8) {\n");
+      cb.append("throw new java.io.IOException(\"Double is exactly 8 bytes."+
+          " Provided buffer is smaller.\");\n");
+      cb.append("}\n");
+      cb.append("double d1 = org.apache.hadoop.record.Utils.readDouble(b1, s1);\n");
+      cb.append("double d2 = org.apache.hadoop.record.Utils.readDouble(b2, s2);\n");
+      cb.append("if (d1 != d2) {\n");
+      cb.append("return ((d1-d2) < 0) ? -1 : 0;\n");
+      cb.append("}\n");
+      cb.append("s1+=8; s2+=8; l1-=8; l2-=8;\n");
+      cb.append("}\n");
     }
+  }
+  /** Creates a new instance of JDouble */
+  public JDouble() {
+    setJavaType(new JavaDouble());
+    setCppType(new CppType("double"));
+    setCType(new CType());
+  }
+  
+  String getSignature() {
+    return "d";
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java Wed Feb 28 19:47:27 2007
@@ -19,85 +19,28 @@
 package org.apache.hadoop.record.compiler;
 
 /**
+ * A thin wrappper around record field.
  *
  * @author Milind Bhandarkar
  */
-public class JField {
-    private JType mType;
-    private String mName;
-    /**
-     * Creates a new instance of JField
-     */
-    public JField(JType type, String name) {
-        mType = type;
-        mName = name;
-    }
-    
-    public String getSignature() {
-        return mType.getSignature();
-    }
-    
-    public String genCppDecl() {
-        return mType.genCppDecl(mName);
-    }
-    
-    public String genJavaDecl() {
-        return mType.genJavaDecl(mName);
-    }
-    
-    public String genJavaConstructorParam(int fIdx) {
-        return mType.genJavaConstructorParam(fIdx);
-    }
-    
-    public String getName() {
-        return "m"+mName;
-    }
-    
-    public String getTag() {
-        return mName;
-    }
-    
-    public JType getType() {
-        return mType;
-    }
-    
-    public String genCppGetSet(int fIdx) {
-        return mType.genCppGetSet(mName, fIdx);
-    }
-    
-    public String genJavaGetSet(int fIdx) {
-        return mType.genJavaGetSet(mName, fIdx);
-    }
-    
-    public String genJavaWriteMethodName() {
-        return mType.genJavaWriteMethod(getName(), getTag());
-    }
-    
-    public String genJavaReadMethodName() {
-        return mType.genJavaReadMethod(getName(), getTag());
-    }
-    
-    public String genJavaCompareTo() {
-        return mType.genJavaCompareTo(getName(), "peer."+getName());
-    }
-    
-    public String genJavaSlurpBytes(String b, String s, String l) {
-      return mType.genJavaSlurpBytes(b,s,l);
-    }
-    
-    public String genJavaCompareBytes() {
-      return mType.genJavaCompareBytes();
-    }
-    
-    public String genJavaEquals() {
-        return mType.genJavaEquals(getName(), "peer."+getName());
-    }
-    
-    public String genJavaHashCode() {
-        return mType.genJavaHashCode(getName());
-    }
-
-    public String genJavaConstructorSet(int fIdx) {
-        return mType.genJavaConstructorSet(mName, fIdx);
-    }
+public class JField<T> {
+  
+  private String name;
+  private T type;
+  
+  /**
+   * Creates a new instance of JField
+   */
+  public JField(String name, T type) {
+    this.type = type;
+    this.name = name;
+  }
+  
+  String getName() {
+    return name;
+  }
+  
+  T getType() {
+    return type;
+  }
 }

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java?view=diff&rev=513122&r1=513121&r2=513122
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFile.java Wed Feb 28 19:47:27 2007
@@ -29,10 +29,12 @@
  * @author Milind Bhandarkar
  */
 public class JFile {
-    
+    /** Possibly full name of the file */
     private String mName;
-    private ArrayList mInclFiles;
-    private ArrayList mRecords;
+    /** Ordered list of included files */
+    private ArrayList<JFile> mInclFiles;
+    /** Ordered list of records declared in this file */
+    private ArrayList<JRecord> mRecords;
     
     /** Creates a new instance of JFile
      *
@@ -40,7 +42,8 @@
      * @param inclFiles included files (as JFile)
      * @param recList List of records defined within this file
      */
-    public JFile(String name, ArrayList inclFiles, ArrayList recList) {
+    public JFile(String name, ArrayList<JFile> inclFiles,
+        ArrayList<JRecord> recList) {
         mName = name;
         mInclFiles = inclFiles;
         mRecords = recList;
@@ -55,17 +58,15 @@
     /** Generate record code in given language. Language should be all
      *  lowercase.
      */
-    public int genCode(String language, String destDir) throws IOException {
-        if ("c++".equals(language)) {
-            CppGenerator gen = new CppGenerator(mName, mInclFiles, mRecords, destDir);
-            gen.genCode();
-        } else if ("java".equals(language)) {
-            JavaGenerator gen = new JavaGenerator(mName, mInclFiles, mRecords, destDir);
-            gen.genCode();
-        } else {
-            System.err.println("Cannnot recognize language:"+language);
-            return 1;
-        }
-        return 0;
+    public int genCode(String language, String destDir, ArrayList<String> options)
+    throws IOException {
+      CodeGenerator gen = CodeGenerator.get(language);
+      if (gen != null) {
+        gen.genCode(mName, mInclFiles, mRecords, destDir, options);
+      } else {
+        System.err.println("Cannnot recognize language:"+language);
+        return 1;
+      }
+      return 0;
     }
 }



Mime
View raw message