hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tomwh...@apache.org
Subject svn commit: r519280 - in /lucene/hadoop/trunk/src: java/org/apache/hadoop/record/ test/org/apache/hadoop/record/
Date Sat, 17 Mar 2007 10:16:56 GMT
Author: tomwhite
Date: Sat Mar 17 03:16:54 2007
New Revision: 519280

URL: http://svn.apache.org/viewvc?view=rev&rev=519280
Log:
Fix incomplete commit of HADOOP-1096.

Added:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordInput.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordOutput.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordInput.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordOutput.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordInput.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordOutput.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordInput.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordOutput.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/RecordBench.java
Removed:
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordInput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordInput.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordInput.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordInput.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.io.DataInputStream;
+import java.io.InputStream;
+
+/**
+ *
+ * @author Milind Bhandarkar
+ */
+public class BinaryRecordInput implements RecordInput {
+    
+    final private DataInput in;
+    
+    static private class BinaryIndex implements Index {
+        private int nelems;
+        private BinaryIndex(int nelems) {
+          this.nelems = nelems;
+        }
+        public boolean done() {
+            return (nelems <= 0);
+        }
+        public void incr() {
+            nelems--;
+        }
+    }
+    
+    /** Creates a new instance of BinaryRecordInput */
+    public BinaryRecordInput(InputStream strm) {
+        this.in = new DataInputStream(strm);
+    }
+    
+    /** Creates a new instance of BinaryRecordInput */
+    public BinaryRecordInput(DataInput din) {
+        this.in = din;
+    }
+    
+    public byte readByte(final String tag) throws IOException {
+        return in.readByte();
+    }
+    
+    public boolean readBool(final String tag) throws IOException {
+        return in.readBoolean();
+    }
+    
+    public int readInt(final String tag) throws IOException {
+        return Utils.readVInt(in);
+    }
+    
+    public long readLong(final String tag) throws IOException {
+        return Utils.readVLong(in);
+    }
+    
+    public float readFloat(final String tag) throws IOException {
+        return in.readFloat();
+    }
+    
+    public double readDouble(final String tag) throws IOException {
+        return in.readDouble();
+    }
+    
+    public String readString(final String tag) throws IOException {
+      return Utils.fromBinaryString(in);
+    }
+    
+    public Buffer readBuffer(final String tag) throws IOException {
+      final int len = Utils.readVInt(in);
+      final byte[] barr = new byte[len];
+      in.readFully(barr);
+      return new Buffer(barr);
+    }
+    
+    public void startRecord(final String tag) throws IOException {
+      // no-op
+    }
+    
+    public void endRecord(final String tag) throws IOException {
+      // no-op
+    }
+    
+    public Index startVector(final String tag) throws IOException {
+      return new BinaryIndex(readInt(tag));
+    }
+    
+    public void endVector(final String tag) throws IOException {
+      // no-op
+}
+    
+    public Index startMap(final String tag) throws IOException {
+        return new BinaryIndex(readInt(tag));
+    }
+    
+    public void endMap(final String tag) throws IOException {
+      // no-op
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordOutput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordOutput.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordOutput.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryRecordOutput.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.util.TreeMap;
+import java.util.ArrayList;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.OutputStream;
+
+/**
+ *
+ * @author Milind Bhandarkar
+ */
+public class BinaryRecordOutput implements RecordOutput {
+    
+    private DataOutput out;
+    
+    /** Creates a new instance of BinaryRecordOutput */
+    public BinaryRecordOutput(OutputStream out) {
+        this.out = new DataOutputStream(out);
+    }
+    
+    /** Creates a new instance of BinaryRecordOutput */
+    public BinaryRecordOutput(DataOutput out) {
+        this.out = out;
+    }
+    
+    
+    public void writeByte(byte b, String tag) throws IOException {
+        out.writeByte(b);
+    }
+    
+    public void writeBool(boolean b, String tag) throws IOException {
+        out.writeBoolean(b);
+    }
+    
+    public void writeInt(int i, String tag) throws IOException {
+        Utils.writeVInt(out, i);
+    }
+    
+    public void writeLong(long l, String tag) throws IOException {
+        Utils.writeVLong(out, l);
+    }
+    
+    public void writeFloat(float f, String tag) throws IOException {
+        out.writeFloat(f);
+    }
+    
+    public void writeDouble(double d, String tag) throws IOException {
+        out.writeDouble(d);
+    }
+    
+    public void writeString(String s, String tag) throws IOException {
+      Utils.toBinaryString(out, s);
+    }
+    
+    public void writeBuffer(Buffer buf, String tag)
+    throws IOException {
+      byte[] barr = buf.get();
+      int len = buf.getCount();
+      Utils.writeVInt(out, len);
+      out.write(barr, 0, len);
+    }
+    
+    public void startRecord(Record r, String tag) throws IOException {}
+    
+    public void endRecord(Record r, String tag) throws IOException {}
+    
+    public void startVector(ArrayList v, String tag) throws IOException {
+        writeInt(v.size(), tag);
+    }
+    
+    public void endVector(ArrayList v, String tag) throws IOException {}
+    
+    public void startMap(TreeMap v, String tag) throws IOException {
+        writeInt(v.size(), tag);
+    }
+    
+    public void endMap(TreeMap v, String tag) throws IOException {}
+    
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordInput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordInput.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordInput.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordInput.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,202 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.InputStreamReader;
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.PushbackReader;
+import java.io.UnsupportedEncodingException;
+
+/**
+ *
+ * @author Milind Bhandarkar
+ */
+public class CsvRecordInput implements RecordInput {
+    
+    private PushbackReader stream;
+    
+    private class CsvIndex implements Index {
+        public boolean done() {
+            char c = '\0';
+            try {
+                c = (char) stream.read();
+                stream.unread(c);
+            } catch (IOException ex) {
+            }
+            return (c == '}') ? true : false;
+        }
+        public void incr() {}
+    }
+    
+    private void throwExceptionOnError(String tag) throws IOException {
+        throw new IOException("Error deserializing "+tag);
+    }
+    
+    private String readField(String tag) throws IOException {
+        try {
+            StringBuffer buf = new StringBuffer();
+            while (true) {
+                char c = (char) stream.read();
+                switch (c) {
+                    case ',':
+                        return buf.toString();
+                    case '}':
+                    case '\n':
+                    case '\r':
+                        stream.unread(c);
+                        return buf.toString();
+                    default:
+                        buf.append(c);
+                }
+            }
+        } catch (IOException ex) {
+            throw new IOException("Error reading "+tag);
+        }
+    }
+    
+    /** Creates a new instance of CsvRecordInput */
+    public CsvRecordInput(InputStream in) {
+      try {
+      stream = new PushbackReader(new InputStreamReader(in, "UTF-8"));
+      } catch (UnsupportedEncodingException ex) {
+        throw new RuntimeException(ex);
+      }
+    }
+    
+    public byte readByte(String tag) throws IOException {
+        return (byte) readLong(tag);
+    }
+    
+    public boolean readBool(String tag) throws IOException {
+        String sval = readField(tag);
+        return "T".equals(sval) ? true : false;
+    }
+    
+    public int readInt(String tag) throws IOException {
+        return (int) readLong(tag);
+    }
+    
+    public long readLong(String tag) throws IOException {
+        String sval = readField(tag);
+        try {
+            long lval = Long.parseLong(sval);
+            return lval;
+        } catch (NumberFormatException ex) {
+            throw new IOException("Error deserializing "+tag);
+        }
+    }
+    
+    public float readFloat(String tag) throws IOException {
+        return (float) readDouble(tag);
+    }
+    
+    public double readDouble(String tag) throws IOException {
+        String sval = readField(tag);
+        try {
+            double dval = Double.parseDouble(sval);
+            return dval;
+        } catch (NumberFormatException ex) {
+            throw new IOException("Error deserializing "+tag);
+        }
+    }
+    
+    public String readString(String tag) throws IOException {
+        String sval = readField(tag);
+        return Utils.fromCSVString(sval);
+    }
+    
+    public Buffer readBuffer(String tag) throws IOException {
+        String sval = readField(tag);
+        return Utils.fromCSVBuffer(sval);
+    }
+    
+    public void startRecord(String tag) throws IOException {
+        if (tag != null && !"".equals(tag)) {
+            char c1 = (char) stream.read();
+            char c2 = (char) stream.read();
+            if (c1 != 's' || c2 != '{') {
+                throw new IOException("Error deserializing "+tag);
+            }
+        }
+    }
+    
+    public void endRecord(String tag) throws IOException {
+        char c = (char) stream.read();
+        if (tag == null || "".equals(tag)) {
+            if (c != '\n' && c != '\r') {
+                throw new IOException("Error deserializing record.");
+            } else {
+                return;
+            }
+        }
+        
+        if (c != '}') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        c = (char) stream.read();
+        if (c != ',') {
+            stream.unread(c);
+        }
+        
+        return;
+    }
+    
+    public Index startVector(String tag) throws IOException {
+        char c1 = (char) stream.read();
+        char c2 = (char) stream.read();
+        if (c1 != 'v' || c2 != '{') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        return new CsvIndex();
+    }
+    
+    public void endVector(String tag) throws IOException {
+        char c = (char) stream.read();
+        if (c != '}') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        c = (char) stream.read();
+        if (c != ',') {
+            stream.unread(c);
+        }
+        return;
+    }
+    
+    public Index startMap(String tag) throws IOException {
+        char c1 = (char) stream.read();
+        char c2 = (char) stream.read();
+        if (c1 != 'm' || c2 != '{') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        return new CsvIndex();
+    }
+    
+    public void endMap(String tag) throws IOException {
+        char c = (char) stream.read();
+        if (c != '}') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        c = (char) stream.read();
+        if (c != ',') {
+            stream.unread(c);
+        }
+        return;
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordOutput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordOutput.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordOutput.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvRecordOutput.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.util.TreeMap;
+import java.util.ArrayList;
+import java.io.PrintStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+
+/**
+ *
+ * @author Milind Bhandarkar
+ */
+public class CsvRecordOutput implements RecordOutput {
+
+    private PrintStream stream;
+    private boolean isFirst = true;
+    
+    private void throwExceptionOnError(String tag) throws IOException {
+        if (stream.checkError()) {
+            throw new IOException("Error serializing "+tag);
+        }
+    }
+ 
+    private void printCommaUnlessFirst() {
+        if (!isFirst) {
+            stream.print(",");
+        }
+        isFirst = false;
+    }
+    
+    /** Creates a new instance of CsvRecordOutput */
+    public CsvRecordOutput(OutputStream out) {
+      try {
+        stream = new PrintStream(out, true, "UTF-8");
+      } catch (UnsupportedEncodingException ex) {
+        throw new RuntimeException(ex);
+      }
+    }
+    
+    public void writeByte(byte b, String tag) throws IOException {
+        writeLong((long)b, tag);
+    }
+    
+    public void writeBool(boolean b, String tag) throws IOException {
+        printCommaUnlessFirst();
+        String val = b ? "T" : "F";
+        stream.print(val);
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeInt(int i, String tag) throws IOException {
+        writeLong((long)i, tag);
+    }
+    
+    public void writeLong(long l, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print(l);
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeFloat(float f, String tag) throws IOException {
+        writeDouble((double)f, tag);
+    }
+    
+    public void writeDouble(double d, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print(d);
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeString(String s, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print(Utils.toCSVString(s));
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeBuffer(Buffer buf, String tag)
+    throws IOException {
+        printCommaUnlessFirst();
+        stream.print(Utils.toCSVBuffer(buf));
+        throwExceptionOnError(tag);
+    }
+    
+    public void startRecord(Record r, String tag) throws IOException {
+        if (tag != null && !"".equals(tag)) {
+            printCommaUnlessFirst();
+            stream.print("s{");
+            isFirst = true;
+        }
+    }
+    
+    public void endRecord(Record r, String tag) throws IOException {
+        if (tag == null || "".equals(tag)) {
+            stream.print("\n");
+            isFirst = true;
+        } else {
+            stream.print("}");
+            isFirst = false;
+        }
+    }
+    
+    public void startVector(ArrayList v, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print("v{");
+        isFirst = true;
+    }
+    
+    public void endVector(ArrayList v, String tag) throws IOException {
+        stream.print("}");
+        isFirst = false;
+    }
+    
+    public void startMap(TreeMap v, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print("m{");
+        isFirst = true;
+    }
+    
+    public void endMap(TreeMap v, String tag) throws IOException {
+        stream.print("}");
+        isFirst = false;
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordInput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordInput.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordInput.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordInput.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+
+/**
+ * Interface that all the Deserializers have to implement.
+ *
+ * @author Milind Bhandarkar
+ */
+public interface RecordInput {
+  /**
+   * Read a byte from serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return value read from serialized record.
+   */
+  byte readByte(String tag) throws IOException;
+  
+  /**
+   * Read a boolean from serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return value read from serialized record.
+   */
+  boolean readBool(String tag) throws IOException;
+  
+  /**
+   * Read an integer from serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return value read from serialized record.
+   */
+  int readInt(String tag) throws IOException;
+  
+  /**
+   * Read a long integer from serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return value read from serialized record.
+   */
+  long readLong(String tag) throws IOException;
+  
+  /**
+   * Read a single-precision float from serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return value read from serialized record.
+   */
+  float readFloat(String tag) throws IOException;
+  
+  /**
+   * Read a double-precision number from serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return value read from serialized record.
+   */
+  double readDouble(String tag) throws IOException;
+  
+  /**
+   * Read a UTF-8 encoded string from serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return value read from serialized record.
+   */
+  String readString(String tag) throws IOException;
+  
+  /**
+   * Read byte array from serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return value read from serialized record.
+   */
+  Buffer readBuffer(String tag) throws IOException;
+  
+  /**
+   * Check the mark for start of the serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   */
+  void startRecord(String tag) throws IOException;
+  
+  /**
+   * Check the mark for end of the serialized record.
+   * @param tag Used by tagged serialization formats (such as XML)
+   */
+  void endRecord(String tag) throws IOException;
+  
+  /**
+   * Check the mark for start of the serialized vector.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return Index that is used to count the number of elements.
+   */
+  Index startVector(String tag) throws IOException;
+  
+  /**
+   * Check the mark for end of the serialized vector.
+   * @param tag Used by tagged serialization formats (such as XML)
+   */
+  void endVector(String tag) throws IOException;
+  
+  /**
+   * Check the mark for start of the serialized map.
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @return Index that is used to count the number of map entries.
+   */
+  Index startMap(String tag) throws IOException;
+  
+  /**
+   * Check the mark for end of the serialized map.
+   * @param tag Used by tagged serialization formats (such as XML)
+   */
+  void endMap(String tag) throws IOException;
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordOutput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordOutput.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordOutput.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordOutput.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.util.TreeMap;
+import java.util.ArrayList;
+
+/**
+ * Interface that alll the serializers have to implement.
+ *
+ * @author Milind Bhandarkar
+ */
+public interface RecordOutput {
+  /**
+   * Write a byte to serialized record.
+   * @param b Byte to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void writeByte(byte b, String tag) throws IOException;
+  
+  /**
+   * Write a boolean to serialized record.
+   * @param b Boolean to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void writeBool(boolean b, String tag) throws IOException;
+  
+  /**
+   * Write an integer to serialized record.
+   * @param i Integer to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void writeInt(int i, String tag) throws IOException;
+  
+  /**
+   * Write a long integer to serialized record.
+   * @param l Long to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void writeLong(long l, String tag) throws IOException;
+  
+  /**
+   * Write a single-precision float to serialized record.
+   * @param f Float to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void writeFloat(float f, String tag) throws IOException;
+  
+  /**
+   * Write a double precision floating point number to serialized record.
+   * @param d Double to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void writeDouble(double d, String tag) throws IOException;
+  
+  /**
+   * Write a unicode string to serialized record.
+   * @param s String to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void writeString(String s, String tag) throws IOException;
+  
+  /**
+   * Write a buffer to serialized record.
+   * @param buf Buffer to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void writeBuffer(Buffer buf, String tag)
+  throws IOException;
+  
+  /**
+   * Mark the start of a record to be serialized.
+   * @param r Record to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void startRecord(Record r, String tag) throws IOException;
+  
+  /**
+   * Mark the end of a serialized record.
+   * @param r Record to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void endRecord(Record r, String tag) throws IOException;
+  
+  /**
+   * Mark the start of a vector to be serialized.
+   * @param v Vector to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void startVector(ArrayList v, String tag) throws IOException;
+  
+  /**
+   * Mark the end of a serialized vector.
+   * @param v Vector to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void endVector(ArrayList v, String tag) throws IOException;
+  
+  /**
+   * Mark the start of a map to be serialized.
+   * @param m Map to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void startMap(TreeMap m, String tag) throws IOException;
+  
+  /**
+   * Mark the end of a serialized map.
+   * @param m Map to be serialized
+   * @param tag Used by tagged serialization formats (such as XML)
+   * @throws IOException Indicates error in serialization
+   */
+  public void endMap(TreeMap m, String tag) throws IOException;
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordInput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordInput.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordInput.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordInput.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+
+import org.xml.sax.*;
+import org.xml.sax.helpers.DefaultHandler;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+
+/**
+ *
+ * @author Milind Bhandarkar
+ */
+class XmlRecordInput implements RecordInput {
+    
+    static private class Value {
+        private String type;
+        private StringBuffer sb;
+        
+        public Value(String t) {
+            type = t;
+            sb = new StringBuffer();
+        }
+        public void addChars(char[] buf, int offset, int len) {
+            sb.append(buf, offset, len);
+        }
+        public String getValue() { return sb.toString(); }
+        public String getType() { return type; }
+    }
+    
+    private class XMLParser extends DefaultHandler {
+        private boolean charsValid = false;
+        
+        private ArrayList valList;
+        
+        private XMLParser(ArrayList vlist) {
+            valList = vlist;
+        }
+        
+        public void startDocument() throws SAXException {}
+        
+        public void endDocument() throws SAXException {}
+        
+        public void startElement(String ns,
+                String sname,
+                String qname,
+                Attributes attrs) throws SAXException {
+            charsValid = false;
+            if ("boolean".equals(qname) ||
+                    "i4".equals(qname) ||
+                    "int".equals(qname) ||
+                    "string".equals(qname) ||
+                    "double".equals(qname) ||
+                    "ex:i1".equals(qname) ||
+                    "ex:i8".equals(qname) ||
+                    "ex:float".equals(qname)) {
+                charsValid = true;
+                valList.add(new Value(qname));
+            } else if ("struct".equals(qname) ||
+                "array".equals(qname)) {
+                valList.add(new Value(qname));
+            }
+        }
+        
+        public void endElement(String ns,
+                String sname,
+                String qname) throws SAXException {
+            charsValid = false;
+            if ("struct".equals(qname) ||
+                    "array".equals(qname)) {
+                valList.add(new Value("/"+qname));
+            }
+        }
+        
+        public void characters(char buf[], int offset, int len)
+        throws SAXException {
+            if (charsValid) {
+                Value v = (Value) valList.get(valList.size()-1);
+                v.addChars(buf, offset,len);
+            }
+        }
+        
+    }
+    
+    private class XmlIndex implements Index {
+        public boolean done() {
+            Value v = (Value) valList.get(vIdx);
+            if ("/array".equals(v.getType())) {
+                valList.set(vIdx, null);
+                vIdx++;
+                return true;
+            } else {
+                return false;
+            }
+        }
+        public void incr() {}
+    }
+    
+    private ArrayList valList;
+    private int vLen;
+    private int vIdx;
+    
+    private Value next() throws IOException {
+        if (vIdx < vLen) {
+            Value v = (Value) valList.get(vIdx);
+            valList.set(vIdx, null);
+            vIdx++;
+            return v;
+        } else {
+            throw new IOException("Error in deserialization.");
+        }
+    }
+    
+    /** Creates a new instance of XmlRecordInput */
+    public XmlRecordInput(InputStream in) {
+      try{
+        valList = new ArrayList();
+        DefaultHandler handler = new XMLParser(valList);
+        SAXParserFactory factory = SAXParserFactory.newInstance();
+        SAXParser parser = factory.newSAXParser();
+        parser.parse(in, handler);
+        vLen = valList.size();
+        vIdx = 0;
+      } catch (Exception ex) {
+        throw new RuntimeException(ex);
+      }
+    }
+    
+    public byte readByte(String tag) throws IOException {
+        Value v = next();
+        if (!"ex:i1".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Byte.parseByte(v.getValue());
+    }
+    
+    public boolean readBool(String tag) throws IOException {
+        Value v = next();
+        if (!"boolean".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return "1".equals(v.getValue());
+    }
+    
+    public int readInt(String tag) throws IOException {
+        Value v = next();
+        if (!"i4".equals(v.getType()) &&
+                !"int".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Integer.parseInt(v.getValue());
+    }
+    
+    public long readLong(String tag) throws IOException {
+        Value v = next();
+        if (!"ex:i8".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Long.parseLong(v.getValue());
+    }
+    
+    public float readFloat(String tag) throws IOException {
+        Value v = next();
+        if (!"ex:float".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Float.parseFloat(v.getValue());
+    }
+    
+    public double readDouble(String tag) throws IOException {
+        Value v = next();
+        if (!"double".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Double.parseDouble(v.getValue());
+    }
+    
+    public String readString(String tag) throws IOException {
+        Value v = next();
+        if (!"string".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Utils.fromXMLString(v.getValue());
+    }
+    
+    public Buffer readBuffer(String tag) throws IOException {
+        Value v = next();
+        if (!"string".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Utils.fromXMLBuffer(v.getValue());
+    }
+    
+    public void startRecord(String tag) throws IOException {
+        Value v = next();
+        if (!"struct".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+    }
+    
+    public void endRecord(String tag) throws IOException {
+        Value v = next();
+        if (!"/struct".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+    }
+    
+    public Index startVector(String tag) throws IOException {
+        Value v = next();
+        if (!"array".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return new XmlIndex();
+    }
+    
+    public void endVector(String tag) throws IOException {}
+    
+    public Index startMap(String tag) throws IOException {
+        return startVector(tag);
+    }
+    
+    public void endMap(String tag) throws IOException { endVector(tag); }
+
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordOutput.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordOutput.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordOutput.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlRecordOutput.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,249 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.util.TreeMap;
+import java.util.ArrayList;
+import java.io.PrintStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.util.Stack;
+
+/**
+ *
+ * @author Milind Bhandarkar
+ */
+class XmlRecordOutput implements RecordOutput {
+
+    private PrintStream stream;
+    
+    private int indent = 0;
+    
+    private Stack compoundStack;
+    
+    private void putIndent() {
+        StringBuffer sb = new StringBuffer("");
+        for (int idx = 0; idx < indent; idx++) {
+            sb.append("  ");
+        }
+        stream.print(sb.toString());
+    }
+    
+    private void addIndent() {
+        indent++;
+    }
+    
+    private void closeIndent() {
+        indent--;
+    }
+    
+    private void printBeginEnvelope(String tag) {
+        if (!compoundStack.empty()) {
+            String s = (String) compoundStack.peek();
+            if ("struct".equals(s)) {
+                putIndent();
+                stream.print("<member>\n");
+                addIndent();
+                putIndent();
+                stream.print("<name>"+tag+"</name>\n");
+                putIndent();
+                stream.print("<value>");
+            } else if ("vector".equals(s)) {
+                stream.print("<value>");
+            } else if ("map".equals(s)) {
+                stream.print("<value>");
+            }
+        } else {
+            stream.print("<value>");
+        }
+    }
+    
+    private void printEndEnvelope(String tag) {
+        if (!compoundStack.empty()) {
+            String s = (String) compoundStack.peek();
+            if ("struct".equals(s)) {
+                stream.print("</value>\n");
+                closeIndent();
+                putIndent();
+                stream.print("</member>\n");
+            } else if ("vector".equals(s)) {
+                stream.print("</value>\n");
+            } else if ("map".equals(s)) {
+                stream.print("</value>\n");
+            }
+        } else {
+            stream.print("</value>\n");
+        }
+    }
+    
+    private void insideVector(String tag) {
+        printBeginEnvelope(tag);
+        compoundStack.push("vector");
+    }
+    
+    private void outsideVector(String tag) throws IOException {
+        String s = (String) compoundStack.pop();
+        if (!"vector".equals(s)) {
+            throw new IOException("Error serializing vector.");
+        }
+        printEndEnvelope(tag);
+    }
+    
+    private void insideMap(String tag) {
+        printBeginEnvelope(tag);
+        compoundStack.push("map");
+    }
+    
+    private void outsideMap(String tag) throws IOException {
+        String s = (String) compoundStack.pop();
+        if (!"map".equals(s)) {
+            throw new IOException("Error serializing map.");
+        }
+        printEndEnvelope(tag);
+    }
+    
+    private void insideRecord(String tag) {
+        printBeginEnvelope(tag);
+        compoundStack.push("struct");
+    }
+    
+    private void outsideRecord(String tag) throws IOException {
+        String s = (String) compoundStack.pop();
+        if (!"struct".equals(s)) {
+            throw new IOException("Error serializing record.");
+        }
+        printEndEnvelope(tag);
+    }
+    
+    /** Creates a new instance of XmlRecordOutput */
+    public XmlRecordOutput(OutputStream out) {
+      try {
+        stream = new PrintStream(out, true, "UTF-8");
+        compoundStack = new Stack();
+      } catch (UnsupportedEncodingException ex) {
+        throw new RuntimeException(ex);
+      }
+    }
+    
+    public void writeByte(byte b, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<ex:i1>");
+        stream.print(Byte.toString(b));
+        stream.print("</ex:i1>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeBool(boolean b, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<boolean>");
+        stream.print(b ? "1" : "0");
+        stream.print("</boolean>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeInt(int i, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<i4>");
+        stream.print(Integer.toString(i));
+        stream.print("</i4>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeLong(long l, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<ex:i8>");
+        stream.print(Long.toString(l));
+        stream.print("</ex:i8>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeFloat(float f, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<ex:float>");
+        stream.print(Float.toString(f));
+        stream.print("</ex:float>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeDouble(double d, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<double>");
+        stream.print(Double.toString(d));
+        stream.print("</double>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeString(String s, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<string>");
+        stream.print(Utils.toXMLString(s));
+        stream.print("</string>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeBuffer(Buffer buf, String tag)
+    throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<string>");
+        stream.print(Utils.toXMLBuffer(buf));
+        stream.print("</string>");
+        printEndEnvelope(tag);
+    }
+    
+    public void startRecord(Record r, String tag) throws IOException {
+        insideRecord(tag);
+        stream.print("<struct>\n");
+        addIndent();
+    }
+    
+    public void endRecord(Record r, String tag) throws IOException {
+        closeIndent();
+        putIndent();
+        stream.print("</struct>");
+        outsideRecord(tag);
+    }
+    
+    public void startVector(ArrayList v, String tag) throws IOException {
+        insideVector(tag);
+        stream.print("<array>\n");
+        addIndent();
+    }
+    
+    public void endVector(ArrayList v, String tag) throws IOException {
+        closeIndent();
+        putIndent();
+        stream.print("</array>");
+        outsideVector(tag);
+    }
+    
+    public void startMap(TreeMap v, String tag) throws IOException {
+        insideMap(tag);
+        stream.print("<array>\n");
+        addIndent();
+    }
+    
+    public void endMap(TreeMap v, String tag) throws IOException {
+        closeIndent();
+        putIndent();
+        stream.print("</array>");
+        outsideMap(tag);
+    }
+
+}

Added: lucene/hadoop/trunk/src/test/org/apache/hadoop/record/RecordBench.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/test/org/apache/hadoop/record/RecordBench.java?view=auto&rev=519280
==============================================================================
--- lucene/hadoop/trunk/src/test/org/apache/hadoop/record/RecordBench.java (added)
+++ lucene/hadoop/trunk/src/test/org/apache/hadoop/record/RecordBench.java Sat Mar 17 03:16:54 2007
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.lang.reflect.Array;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Random;
+
+/**
+ * Benchmark for various types of serializations
+ * @author milindb
+ */
+public class RecordBench {
+  
+  private static class Times {
+    long init;
+    long serialize;
+    long deserialize;
+    long write;
+    long readFields;
+  };
+  
+  private static final long SEED = 0xDEADBEEFL;
+  private static final Random rand = new Random();
+  
+  /** Do not allow to create a new instance of RecordBench */
+  private RecordBench() {}
+  
+  private static void initBuffers(Record[] buffers) {
+    final int BUFLEN = 32;
+    for (int idx = 0; idx < buffers.length; idx++) {
+      buffers[idx] = new RecBuffer();
+      int buflen = rand.nextInt(BUFLEN);
+      byte[] bytes = new byte[buflen];
+      rand.nextBytes(bytes);
+      ((RecBuffer)buffers[idx]).setData(new Buffer(bytes));
+    }
+  }
+  
+  private static void initStrings(Record[] strings) {
+    final int STRLEN = 32;
+    for (int idx = 0; idx < strings.length; idx++) {
+      strings[idx] = new RecString();
+      int strlen = rand.nextInt(STRLEN);
+      StringBuilder sb = new StringBuilder(strlen);
+      for (int ich = 0; ich < strlen; ich++) {
+        int cpt = 0;
+        while (true) {
+          cpt = rand.nextInt(0x10FFFF+1);
+          if (Utils.isValidCodePoint(cpt)) {
+            break;
+          }
+        }
+        sb.appendCodePoint(cpt);
+      }
+      ((RecString)strings[idx]).setData(sb.toString());
+    }
+  }
+  
+  private static void initInts(Record[] ints) {
+    for (int idx = 0; idx < ints.length; idx++) {
+      ints[idx] = new RecInt();
+      ((RecInt)ints[idx]).setData(rand.nextInt());
+    }
+  }
+  
+  private static Record[] makeArray(String type, int numRecords, Times times) {
+    Method init = null;
+    try {
+      init = RecordBench.class.getDeclaredMethod("init"+
+          toCamelCase(type) + "s",
+          new Class[] {Record[].class});
+    } catch (NoSuchMethodException ex) {
+      throw new RuntimeException(ex);
+    }
+
+    Record[] records = new Record[numRecords];
+    times.init = System.nanoTime();
+    try {
+      init.invoke(null, new Object[]{records});
+    } catch (Exception ex) {
+      throw new RuntimeException(ex);
+    }
+    times.init = System.nanoTime() - times.init;
+    return records;
+  }
+  
+  private static void runBinaryBench(String type, int numRecords, Times times)
+  throws IOException {
+    Record[] records = makeArray(type, numRecords, times);
+    ByteArrayOutputStream bout = new ByteArrayOutputStream();
+    BinaryRecordOutput rout = new BinaryRecordOutput(bout);
+    DataOutputStream dout = new DataOutputStream(bout);
+    
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].serialize(rout);
+    }
+    bout.reset();
+    
+    times.serialize = System.nanoTime();
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].serialize(rout);
+    }
+    times.serialize = System.nanoTime() - times.serialize;
+    
+    byte[] serialized = bout.toByteArray();
+    ByteArrayInputStream bin = new ByteArrayInputStream(serialized);
+    BinaryRecordInput rin = new BinaryRecordInput(bin);
+    
+    times.deserialize = System.nanoTime();
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].deserialize(rin);
+    }
+    times.deserialize = System.nanoTime() - times.deserialize;
+    
+    bout.reset();
+    
+    times.write = System.nanoTime();
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].write(dout);
+    }
+    times.write = System.nanoTime() - times.write;
+    
+    bin.reset();
+    DataInputStream din = new DataInputStream(bin);
+    
+    times.readFields = System.nanoTime();
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].readFields(din);
+    }
+    times.readFields = System.nanoTime() - times.readFields;
+  }
+  
+  private static void runCsvBench(String type, int numRecords, Times times)
+  throws IOException {
+    Record[] records = makeArray(type, numRecords, times);
+    ByteArrayOutputStream bout = new ByteArrayOutputStream();
+    CsvRecordOutput rout = new CsvRecordOutput(bout);
+    
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].serialize(rout);
+    }
+    bout.reset();
+    
+    times.serialize = System.nanoTime();
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].serialize(rout);
+    }
+    times.serialize = System.nanoTime() - times.serialize;
+    
+    byte[] serialized = bout.toByteArray();
+    ByteArrayInputStream bin = new ByteArrayInputStream(serialized);
+    CsvRecordInput rin = new CsvRecordInput(bin);
+    
+    times.deserialize = System.nanoTime();
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].deserialize(rin);
+    }
+    times.deserialize = System.nanoTime() - times.deserialize;
+  }
+  
+  private static void runXmlBench(String type, int numRecords, Times times)
+  throws IOException {
+    Record[] records = makeArray(type, numRecords, times);
+    ByteArrayOutputStream bout = new ByteArrayOutputStream();
+    XmlRecordOutput rout = new XmlRecordOutput(bout);
+    
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].serialize(rout);
+    }
+    bout.reset();
+    
+    bout.write("<records>\n".getBytes());
+    
+    times.serialize = System.nanoTime();
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].serialize(rout);
+    }
+    times.serialize = System.nanoTime() - times.serialize;
+    
+    bout.write("</records>\n".getBytes());
+    
+    byte[] serialized = bout.toByteArray();
+    ByteArrayInputStream bin = new ByteArrayInputStream(serialized);
+        
+    times.deserialize = System.nanoTime();
+    XmlRecordInput rin = new XmlRecordInput(bin);
+    for(int idx = 0; idx < numRecords; idx++) {
+      records[idx].deserialize(rin);
+    }
+    times.deserialize = System.nanoTime() - times.deserialize;
+  }
+
+  private static void printTimes(String type,
+      String format,
+      int numRecords,
+      Times times) {
+    System.out.println("Type: " + type + " Format: " + format +
+        " #Records: "+numRecords);
+    if (times.init != 0) {
+      System.out.println("Initialization Time (Per record) : "+
+          times.init/numRecords + " Nanoseconds");
+    }
+    
+    if (times.serialize != 0) {
+      System.out.println("Serialization Time (Per Record) : "+
+          times.serialize/numRecords + " Nanoseconds");
+    }
+    
+    if (times.deserialize != 0) {
+      System.out.println("Deserialization Time (Per Record) : "+
+          times.deserialize/numRecords + " Nanoseconds");
+    }
+    
+    if (times.write != 0) {
+      System.out.println("Write Time (Per Record) : "+
+          times.write/numRecords + " Nanoseconds");
+    }
+    
+    if (times.readFields != 0) {
+      System.out.println("ReadFields Time (Per Record) : "+
+          times.readFields/numRecords + " Nanoseconds");
+    }
+    
+    System.out.println();
+  }
+  
+  private static String toCamelCase(String inp) {
+    char firstChar = inp.charAt(0);
+    if (Character.isLowerCase(firstChar)) {
+      return ""+Character.toUpperCase(firstChar) + inp.substring(1);
+    }
+    return inp;
+  }
+  
+  private static void exitOnError() {
+    String usage = "RecordBench {buffer|string|int}"+
+        " {binary|csv|xml} <numRecords>";
+    System.out.println(usage);
+    System.exit(1);
+  }
+  
+  /**
+   * @param args the command line arguments
+   */
+  public static void main(String[] args) throws IOException {
+    String version = "RecordBench v0.1";
+    System.out.println(version+"\n");
+    
+    if (args.length != 3) {
+      exitOnError();
+    }
+    
+    String typeName = args[0];
+    String format = args[1];
+    int numRecords = Integer.decode(args[2]).intValue();
+    
+    Method bench = null;
+    try {
+    bench = RecordBench.class.getDeclaredMethod("run"+
+        toCamelCase(format) + "Bench",
+        new Class[] {String.class, Integer.TYPE, Times.class});
+    } catch (NoSuchMethodException ex) {
+      ex.printStackTrace();
+      exitOnError();
+    }
+    
+    if (numRecords < 0) {
+      exitOnError();
+    }
+    
+    // dry run
+    rand.setSeed(SEED);
+    Times times = new Times();
+    try {
+      bench.invoke(null, new Object[] {typeName, numRecords, times});
+    } catch (Exception ex) {
+      ex.printStackTrace();
+      System.exit(1);
+    }
+    
+    // timed run
+    rand.setSeed(SEED);
+    try {
+      bench.invoke(null, new Object[] {typeName, numRecords, times});
+    } catch (Exception ex) {
+      ex.printStackTrace();
+      System.exit(1);
+    }
+    printTimes(typeName, format, numRecords, times);
+  }
+}



Mime
View raw message