hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cutt...@apache.org
Subject svn commit: r399509 [2/5] - in /lucene/hadoop/trunk: ./ bin/ src/c++/ src/c++/librecordio/ src/c++/librecordio/test/ src/java/org/apache/hadoop/record/ src/java/org/apache/hadoop/record/compiler/ src/java/org/apache/hadoop/record/compiler/generated/ sr...
Date Thu, 04 May 2006 02:04:07 GMT
Added: lucene/hadoop/trunk/src/c++/librecordio/xmlarchive.hh
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/c%2B%2B/librecordio/xmlarchive.hh?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/xmlarchive.hh (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/xmlarchive.hh Wed May  3 19:04:01 2006
@@ -0,0 +1,263 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef XMLARCHIVE_HH_
+#define XMLARCHIVE_HH_
+
+#include <xercesc/parsers/SAXParser.hpp>
+#include <xercesc/util/PlatformUtils.hpp>
+#include <xercesc/util/BinInputStream.hpp>
+#include <xercesc/sax/HandlerBase.hpp>
+#include <xercesc/sax/InputSource.hpp>
+#include "recordio.hh"
+
+XERCES_CPP_NAMESPACE_USE
+
+namespace hadoop {
+
+class Value {
+private:
+  std::string type;
+  std::string value;
+public:
+  Value(const std::string& t) { type = t; }
+  void addChars(const char* buf, unsigned int len) {
+    value += std::string(buf, len);
+  }
+  const std::string& getType() const { return type; }
+  const std::string& getValue() const { return value; }
+};
+  
+class MySAXHandler : public HandlerBase {
+private:
+  std::vector<Value>& vlist;
+  bool charsValid;
+public:
+  MySAXHandler(std::vector<Value>& list) : vlist(list) {charsValid = false;}
+  void startElement(const XMLCh* const name, AttributeList& attr);
+  void endElement(const XMLCh* const name);
+  void characters(const XMLCh* const buf, unsigned int len);
+};
+
+class XmlIndex : public Index {
+private:
+  std::vector<Value>& vlist;
+  unsigned int& vidx;
+public:
+  XmlIndex(std::vector<Value>& list, unsigned int& idx) : vlist(list), vidx(idx) {}
+  bool done() {
+   Value v = vlist[vidx];
+   return (v.getType() == "/array") ? true : false;
+  }
+  void incr() {}
+  ~XmlIndex() {} 
+};
+
+class MyBinInputStream : public BinInputStream {
+private:
+  InStream& stream;
+  unsigned int pos;
+public:
+  MyBinInputStream(InStream& s) : stream(s) { pos = 0; }
+  virtual unsigned int curPos() const { return pos; }
+  virtual unsigned int readBytes(XMLByte* const toFill,
+      const unsigned int maxToRead) {
+    ssize_t nread = stream.read(toFill, maxToRead);
+    if (nread < 0) {
+      return 0;
+    } else {
+      pos += nread;
+      return nread;
+    }
+  }
+};
+
+
+class MyInputSource : public InputSource {
+private:
+  InStream& stream;
+public:
+  MyInputSource(InStream& s) : stream(s) {  }
+  virtual BinInputStream* makeStream() const {
+    return new MyBinInputStream(stream);
+  }
+  virtual const XMLCh* getEncoding() const {
+    return XMLString::transcode("UTF-8");
+  }
+  virtual ~MyInputSource() {}
+};
+  
+class IXmlArchive : public IArchive {
+private:
+  std::vector<Value> vlist;
+  unsigned int vidx;
+  MySAXHandler *docHandler;
+  SAXParser *parser;
+  MyInputSource* src;
+  Value next() {
+    Value v = vlist[vidx];
+    vidx++;
+    return v;
+  }
+public:
+  IXmlArchive(InStream& _stream) {
+    vidx = 0;
+    try {
+      XMLPlatformUtils::Initialize();
+    } catch (const XMLException& e) {
+      throw new IOException("Unable to initialize XML Parser.");
+    }
+    parser = new SAXParser();
+    docHandler = new MySAXHandler(vlist);
+    parser->setDocumentHandler(docHandler);
+    src = new MyInputSource(_stream);
+    try {
+      parser->parse(*src);
+    } catch (const XMLException& e) {
+      throw new IOException("Unable to parse XML stream.");
+    } catch (const SAXParseException& e) {
+      throw new IOException("Unable to parse XML stream.");
+    }
+    delete parser;
+    delete docHandler;
+  }
+  virtual void deserialize(int8_t& t, const char* tag);
+  virtual void deserialize(bool& t, const char* tag);
+  virtual void deserialize(int32_t& t, const char* tag);
+  virtual void deserialize(int64_t& t, const char* tag);
+  virtual void deserialize(float& t, const char* tag);
+  virtual void deserialize(double& t, const char* tag);
+  virtual void deserialize(std::string& t, const char* tag);
+  virtual void deserialize(std::string& t, size_t& len, const char* tag);
+  virtual void startRecord(Record& s, const char* tag);
+  virtual void endRecord(Record& s, const char* tag);
+  virtual Index* startVector(const char* tag);
+  virtual void endVector(Index* idx, const char* tag);
+  virtual Index* startMap(const char* tag);
+  virtual void endMap(Index* idx, const char* tag);
+  virtual ~IXmlArchive() {
+    XMLPlatformUtils::Terminate();
+  }
+};
+
+class OXmlArchive : public OArchive {
+private:
+  OutStream& stream;
+  
+  std::vector<std::string> cstack;
+  
+  void insideRecord(const char* tag) {
+    printBeginEnvelope(tag);
+    cstack.push_back("record");
+  }
+  
+  void outsideRecord(const char* tag) {
+    std::string s = cstack.back();
+    cstack.pop_back();
+    if (s != "record") {
+      throw new IOException("Error deserializing record.");
+    }
+    printEndEnvelope(tag);
+  }
+  
+  void insideVector(const char* tag) {
+    printBeginEnvelope(tag);
+    cstack.push_back("vector");
+  }
+  
+  void outsideVector(const char* tag) {
+    std::string s = cstack.back();
+    cstack.pop_back();
+    if (s != "vector") {
+      throw new IOException("Error deserializing vector.");
+    }
+    printEndEnvelope(tag);
+  }
+  
+  void insideMap(const char* tag) {
+    printBeginEnvelope(tag);
+    cstack.push_back("map");
+  }
+  
+  void outsideMap(const char* tag) {
+    std::string s = cstack.back();
+    cstack.pop_back();
+    if (s != "map") {
+      throw new IOException("Error deserializing map.");
+    }
+    printEndEnvelope(tag);
+  }
+  
+  void p(const char* cstr) {
+    stream.write(cstr, strlen(cstr));
+  }
+  
+  void printBeginEnvelope(const char* tag) {
+    if (cstack.size() != 0) {
+      std::string s = cstack.back();
+      if ("record" == s) {
+        p("<member>\n");
+        p("<name>");
+        p(tag);
+        p("</name>\n");
+        p("<value>");
+      } else if ("vector" == s) {
+        p("<value>");
+      } else if ("map" == s) {
+        p("<value>");
+      }
+    } else {
+      p("<value>");
+    }
+  }
+  
+  void printEndEnvelope(const char* tag) {
+    if (cstack.size() != 0) {
+      std::string s = cstack.back();
+      if ("record" == s) {
+        p("</value>\n");
+        p("</member>\n");
+      } else if ("vector" == s) {
+        p("</value>\n");
+      } else if ("map" == s) {
+        p("</value>\n");
+      }
+    } else {
+      p("</value>\n");
+    }
+  }
+  
+public:
+  OXmlArchive(OutStream& _stream) : stream(_stream) {}
+  virtual void serialize(int8_t t, const char* tag);
+  virtual void serialize(bool t, const char* tag);
+  virtual void serialize(int32_t t, const char* tag);
+  virtual void serialize(int64_t t, const char* tag);
+  virtual void serialize(float t, const char* tag);
+  virtual void serialize(double t, const char* tag);
+  virtual void serialize(const std::string& t, const char* tag);
+  virtual void serialize(const std::string& t, size_t len, const char* tag);
+  virtual void startRecord(const Record& s, const char* tag);
+  virtual void endRecord(const Record& s, const char* tag);
+  virtual void startVector(size_t len, const char* tag);
+  virtual void endVector(size_t len, const char* tag);
+  virtual void startMap(size_t len, const char* tag);
+  virtual void endMap(size_t len, const char* tag);
+  virtual ~OXmlArchive();
+};
+
+}
+#endif /*XMLARCHIVE_HH_*/

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryInputArchive.java Wed May  3 19:04:01 2006
@@ -0,0 +1,115 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.DataInput;
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.InputStream;
+
+
+/**
+ *
+ * @author milindb
+ */
+public class BinaryInputArchive implements InputArchive {
+    
+    private DataInput in;
+    
+    static BinaryInputArchive getArchive(InputStream strm) {
+        return new BinaryInputArchive(new DataInputStream(strm));
+    }
+    
+    static private class BinaryIndex implements Index {
+        private int nelems;
+        BinaryIndex(int nelems) {
+            this.nelems = nelems;
+        }
+        public boolean done() {
+            return (nelems <= 0);
+        }
+        public void incr() {
+            nelems--;
+        }
+    }
+    /** Creates a new instance of BinaryInputArchive */
+    public BinaryInputArchive(DataInput in) {
+        this.in = in;
+    }
+    
+    public byte readByte(String tag) throws IOException {
+        return in.readByte();
+    }
+    
+    public boolean readBool(String tag) throws IOException {
+        return in.readBoolean();
+    }
+    
+    public int readInt(String tag) throws IOException {
+        return Utils.readInt(in);
+    }
+    
+    public long readLong(String tag) throws IOException {
+        return Utils.readLong(in);
+    }
+    
+    public float readFloat(String tag) throws IOException {
+        return in.readFloat();
+    }
+    
+    public double readDouble(String tag) throws IOException {
+        return in.readDouble();
+    }
+    
+    public String readString(String tag) throws IOException {
+        int len = Utils.readInt(in);
+        byte[] chars = new byte[len];
+        in.readFully(chars);
+        return new String(chars, "UTF-8");
+    }
+    
+    public ByteArrayOutputStream readBuffer(String tag) throws IOException {
+        int len = Utils.readInt(in);
+        ByteArrayOutputStream buf = new ByteArrayOutputStream(len);
+        byte[] arr = new byte[len];
+        in.readFully(arr);
+        buf.write(arr, 0, len);
+        return buf;
+    }
+    
+    public void readRecord(Record r, String tag) throws IOException {
+        r.deserialize(this, tag);
+    }
+    
+    public void startRecord(String tag) throws IOException {}
+    
+    public void endRecord(String tag) throws IOException {}
+    
+    public Index startVector(String tag) throws IOException {
+        return new BinaryIndex(Utils.readInt(in));
+    }
+    
+    public void endVector(String tag) throws IOException {}
+    
+    public Index startMap(String tag) throws IOException {
+        return new BinaryIndex(Utils.readInt(in));
+    }
+    
+    public void endMap(String tag) throws IOException {}
+    
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/BinaryOutputArchive.java Wed May  3 19:04:01 2006
@@ -0,0 +1,101 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.util.TreeMap;
+import java.util.ArrayList;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.OutputStream;
+
+/**
+ *
+ * @author milindb
+ */
+public class BinaryOutputArchive implements OutputArchive {
+    
+    private DataOutput out;
+    
+    static BinaryOutputArchive getArchive(OutputStream strm) {
+        return new BinaryOutputArchive(new DataOutputStream(strm));
+    }
+    
+    /** Creates a new instance of BinaryOutputArchive */
+    public BinaryOutputArchive(DataOutput out) {
+        this.out = out;
+    }
+    
+    public void writeByte(byte b, String tag) throws IOException {
+        out.writeByte(b);
+    }
+    
+    public void writeBool(boolean b, String tag) throws IOException {
+        out.writeBoolean(b);
+    }
+    
+    public void writeInt(int i, String tag) throws IOException {
+        Utils.writeInt(out, i);
+    }
+    
+    public void writeLong(long l, String tag) throws IOException {
+        Utils.writeLong(out, l);
+    }
+    
+    public void writeFloat(float f, String tag) throws IOException {
+        out.writeFloat(f);
+    }
+    
+    public void writeDouble(double d, String tag) throws IOException {
+        out.writeDouble(d);
+    }
+    
+    public void writeString(String s, String tag) throws IOException {
+        byte[] chars = s.getBytes("UTF-8");
+        Utils.writeInt(out, chars.length);
+        out.write(chars);
+    }
+    
+    public void writeBuffer(ByteArrayOutputStream buf, String tag)
+    throws IOException {
+        byte[] barr = buf.toByteArray();
+        Utils.writeInt(out, barr.length);
+        out.write(barr);
+    }
+    
+    public void writeRecord(Record r, String tag) throws IOException {
+        r.serialize(this, tag);
+    }
+    
+    public void startRecord(Record r, String tag) throws IOException {}
+    
+    public void endRecord(Record r, String tag) throws IOException {}
+    
+    public void startVector(ArrayList v, String tag) throws IOException {
+        Utils.writeInt(out, v.size());
+    }
+    
+    public void endVector(ArrayList v, String tag) throws IOException {}
+    
+    public void startMap(TreeMap v, String tag) throws IOException {
+        Utils.writeInt(out, v.size());
+    }
+    
+    public void endMap(TreeMap v, String tag) throws IOException {}
+    
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvInputArchive.java Wed May  3 19:04:01 2006
@@ -0,0 +1,209 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.InputStreamReader;
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.io.PushbackInputStream;
+import java.io.PushbackReader;
+import java.io.UnsupportedEncodingException;
+
+/**
+ *
+ * @author milindb
+ */
+class CsvInputArchive implements InputArchive {
+    
+    private PushbackReader stream;
+    
+    private class CsvIndex implements Index {
+        public boolean done() {
+            char c = '\0';
+            try {
+                c = (char) stream.read();
+                stream.unread(c);
+            } catch (IOException ex) {
+            }
+            return (c == '}') ? true : false;
+        }
+        public void incr() {}
+    }
+    
+    private void throwExceptionOnError(String tag) throws IOException {
+        throw new IOException("Error deserializing "+tag);
+    }
+    
+    private String readField(String tag) throws IOException {
+        try {
+            StringBuffer buf = new StringBuffer();
+            while (true) {
+                char c = (char) stream.read();
+                switch (c) {
+                    case ',':
+                        return buf.toString();
+                    case '}':
+                    case '\n':
+                    case '\r':
+                        stream.unread(c);
+                        return buf.toString();
+                    default:
+                        buf.append(c);
+                }
+            }
+        } catch (IOException ex) {
+            throw new IOException("Error reading "+tag);
+        }
+    }
+    
+    static CsvInputArchive getArchive(InputStream strm)
+    throws UnsupportedEncodingException {
+        return new CsvInputArchive(strm);
+    }
+    
+    /** Creates a new instance of CsvInputArchive */
+    public CsvInputArchive(InputStream in)
+    throws UnsupportedEncodingException {
+        stream = new PushbackReader(new InputStreamReader(in, "UTF-8"));
+    }
+    
+    public byte readByte(String tag) throws IOException {
+        return (byte) readLong(tag);
+    }
+    
+    public boolean readBool(String tag) throws IOException {
+        String sval = readField(tag);
+        return "T".equals(sval) ? true : false;
+    }
+    
+    public int readInt(String tag) throws IOException {
+        return (int) readLong(tag);
+    }
+    
+    public long readLong(String tag) throws IOException {
+        String sval = readField(tag);
+        try {
+            long lval = Long.parseLong(sval);
+            return lval;
+        } catch (NumberFormatException ex) {
+            throw new IOException("Error deserializing "+tag);
+        }
+    }
+    
+    public float readFloat(String tag) throws IOException {
+        return (float) readDouble(tag);
+    }
+    
+    public double readDouble(String tag) throws IOException {
+        String sval = readField(tag);
+        try {
+            double dval = Double.parseDouble(sval);
+            return dval;
+        } catch (NumberFormatException ex) {
+            throw new IOException("Error deserializing "+tag);
+        }
+    }
+    
+    public String readString(String tag) throws IOException {
+        String sval = readField(tag);
+        return Utils.fromCSVString(sval);
+        
+    }
+    
+    public ByteArrayOutputStream readBuffer(String tag) throws IOException {
+        String sval = readField(tag);
+        return Utils.fromCSVBuffer(sval);
+    }
+    
+    public void readRecord(Record r, String tag) throws IOException {
+        r.deserialize(this, tag);
+    }
+    
+    public void startRecord(String tag) throws IOException {
+        if (tag != null && !"".equals(tag)) {
+            char c1 = (char) stream.read();
+            char c2 = (char) stream.read();
+            if (c1 != 's' || c2 != '{') {
+                throw new IOException("Error deserializing "+tag);
+            }
+        }
+    }
+    
+    public void endRecord(String tag) throws IOException {
+        char c = (char) stream.read();
+        if (tag == null || "".equals(tag)) {
+            if (c != '\n' && c != '\r') {
+                throw new IOException("Error deserializing record.");
+            } else {
+                return;
+            }
+        }
+        
+        if (c != '}') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        c = (char) stream.read();
+        if (c != ',') {
+            stream.unread(c);
+        }
+        
+        return;
+    }
+    
+    public Index startVector(String tag) throws IOException {
+        char c1 = (char) stream.read();
+        char c2 = (char) stream.read();
+        if (c1 != 'v' || c2 != '{') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        return new CsvIndex();
+    }
+    
+    public void endVector(String tag) throws IOException {
+        char c = (char) stream.read();
+        if (c != '}') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        c = (char) stream.read();
+        if (c != ',') {
+            stream.unread(c);
+        }
+        return;
+    }
+    
+    public Index startMap(String tag) throws IOException {
+        char c1 = (char) stream.read();
+        char c2 = (char) stream.read();
+        if (c1 != 'm' || c2 != '{') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        return new CsvIndex();
+    }
+    
+    public void endMap(String tag) throws IOException {
+        char c = (char) stream.read();
+        if (c != '}') {
+            throw new IOException("Error deserializing "+tag);
+        }
+        c = (char) stream.read();
+        if (c != ',') {
+            stream.unread(c);
+        }
+        return;
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/CsvOutputArchive.java Wed May  3 19:04:01 2006
@@ -0,0 +1,147 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.util.TreeMap;
+import java.util.ArrayList;
+import java.io.PrintStream;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+
+/**
+ *
+ * @author milindb
+ */
+public class CsvOutputArchive implements OutputArchive {
+
+    private PrintStream stream;
+    private boolean isFirst = true;
+    
+    static CsvOutputArchive getArchive(OutputStream strm)
+    throws UnsupportedEncodingException {
+        return new CsvOutputArchive(strm);
+    }
+    
+    private void throwExceptionOnError(String tag) throws IOException {
+        if (stream.checkError()) {
+            throw new IOException("Error serializing "+tag);
+        }
+    }
+ 
+    private void printCommaUnlessFirst() {
+        if (!isFirst) {
+            stream.print(",");
+        }
+        isFirst = false;
+    }
+    
+    /** Creates a new instance of CsvOutputArchive */
+    public CsvOutputArchive(OutputStream out)
+    throws UnsupportedEncodingException {
+        stream = new PrintStream(out, true, "UTF-8");
+    }
+    
+    public void writeByte(byte b, String tag) throws IOException {
+        writeLong((long)b, tag);
+    }
+    
+    public void writeBool(boolean b, String tag) throws IOException {
+        printCommaUnlessFirst();
+        String val = b ? "T" : "F";
+        stream.print(val);
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeInt(int i, String tag) throws IOException {
+        writeLong((long)i, tag);
+    }
+    
+    public void writeLong(long l, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print(l);
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeFloat(float f, String tag) throws IOException {
+        writeDouble((double)f, tag);
+    }
+    
+    public void writeDouble(double d, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print(d);
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeString(String s, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print(Utils.toCSVString(s));
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeBuffer(ByteArrayOutputStream buf, String tag)
+    throws IOException {
+        printCommaUnlessFirst();
+        stream.print(Utils.toCSVBuffer(buf));
+        throwExceptionOnError(tag);
+    }
+    
+    public void writeRecord(Record r, String tag) throws IOException {
+        r.serialize(this, tag);
+    }
+    
+    public void startRecord(Record r, String tag) throws IOException {
+        if (tag != null && !"".equals(tag)) {
+            printCommaUnlessFirst();
+            stream.print("s{");
+            isFirst = true;
+        }
+    }
+    
+    public void endRecord(Record r, String tag) throws IOException {
+        if (tag == null || "".equals(tag)) {
+            stream.print("\n");
+            isFirst = true;
+        } else {
+            stream.print("}");
+            isFirst = false;
+        }
+    }
+    
+    public void startVector(ArrayList v, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print("v{");
+        isFirst = true;
+    }
+    
+    public void endVector(ArrayList v, String tag) throws IOException {
+        stream.print("}");
+        isFirst = false;
+    }
+    
+    public void startMap(TreeMap v, String tag) throws IOException {
+        printCommaUnlessFirst();
+        stream.print("m{");
+        isFirst = true;
+    }
+    
+    public void endMap(TreeMap v, String tag) throws IOException {
+        stream.print("}");
+        isFirst = false;
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Index.java Wed May  3 19:04:01 2006
@@ -0,0 +1,37 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+/**
+ * Interface that acts as an iterator for deserializing maps.
+ * The deserializer returns an instance that the record uses to
+ * read vectors and maps. An example of usage is as follows:
+ *
+ * <code>
+ * Index idx = startVector(...);
+ * while (!idx.done()) {
+ *   .... // read element of a vector
+ *   idx.incr();
+ * }
+ * </code>
+ *
+ * @author Milind Bhandarkar
+ */
+public interface Index {
+    public boolean done();
+    public void incr();
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/InputArchive.java Wed May  3 19:04:01 2006
@@ -0,0 +1,45 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.util.TreeMap;
+import java.util.ArrayList;
+
+/**
+ * Interface that all the Deserializers have to implement.
+ *
+ * @author milindb
+ */
+public interface InputArchive {
+    public byte readByte(String tag) throws IOException;
+    public boolean readBool(String tag) throws IOException;
+    public int readInt(String tag) throws IOException;
+    public long readLong(String tag) throws IOException;
+    public float readFloat(String tag) throws IOException;
+    public double readDouble(String tag) throws IOException;
+    public String readString(String tag) throws IOException;
+    public ByteArrayOutputStream readBuffer(String tag) throws IOException;
+    public void readRecord(Record r, String tag) throws IOException;
+    public void startRecord(String tag) throws IOException;
+    public void endRecord(String tag) throws IOException;
+    public Index startVector(String tag) throws IOException;
+    public void endVector(String tag) throws IOException;
+    public Index startMap(String tag) throws IOException;
+    public void endMap(String tag) throws IOException;
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/OutputArchive.java Wed May  3 19:04:01 2006
@@ -0,0 +1,47 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.util.TreeMap;
+import java.util.ArrayList;
+
+/**
+ * Interface that alll the serializers have to implement.
+ *
+ * @author milindb
+ */
+public interface OutputArchive {
+    public void writeByte(byte b, String tag) throws IOException;
+    public void writeBool(boolean b, String tag) throws IOException;
+    public void writeInt(int i, String tag) throws IOException;
+    public void writeLong(long l, String tag) throws IOException;
+    public void writeFloat(float f, String tag) throws IOException;
+    public void writeDouble(double d, String tag) throws IOException;
+    public void writeString(String s, String tag) throws IOException;
+    public void writeBuffer(ByteArrayOutputStream buf, String tag)
+        throws IOException;
+    public void writeRecord(Record r, String tag) throws IOException;
+    public void startRecord(Record r, String tag) throws IOException;
+    public void endRecord(Record r, String tag) throws IOException;
+    public void startVector(ArrayList v, String tag) throws IOException;
+    public void endVector(ArrayList v, String tag) throws IOException;
+    public void startMap(TreeMap v, String tag) throws IOException;
+    public void endMap(TreeMap v, String tag) throws IOException;
+
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Record.java Wed May  3 19:04:01 2006
@@ -0,0 +1,31 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+
+/**
+ * Interface that is implemented by generated classes.
+ * 
+ * @author Milind Bhandarkar
+ */
+public interface Record {
+    public void serialize(OutputArchive archive, String tag)
+        throws IOException;
+    public void deserialize(InputArchive archive, String tag)
+        throws IOException;
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordReader.java Wed May  3 19:04:01 2006
@@ -0,0 +1,96 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.DataInputStream;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import javax.xml.parsers.ParserConfigurationException;
+import org.xml.sax.SAXException;
+
+/**
+ * Front-end interface to deserializers. Also acts as a factory
+ * for deserializers.
+ *
+ * @author milindb
+ */
+public class RecordReader {
+    
+    private InputArchive archive;
+
+    static private HashMap archiveFactory;
+    
+    static {
+        archiveFactory = new HashMap();
+        Class[] params = { InputStream.class };
+        try {
+            archiveFactory.put("binary",
+                    BinaryInputArchive.class.getDeclaredMethod(
+                        "getArchive", params));
+            archiveFactory.put("csv",
+                    CsvInputArchive.class.getDeclaredMethod(
+                        "getArchive", params));
+            archiveFactory.put("xml",
+                    XmlInputArchive.class.getDeclaredMethod(
+                        "getArchive", params));
+        } catch (SecurityException ex) {
+            ex.printStackTrace();
+        } catch (NoSuchMethodException ex) {
+            ex.printStackTrace();
+        }
+    }
+    
+    static private InputArchive createArchive(InputStream in, String format)
+    throws IOException {
+        Method factory = (Method) archiveFactory.get(format);
+        if (factory != null) {
+            Object[] params = { in };
+            try {
+                return (InputArchive) factory.invoke(null, params);
+            } catch (IllegalArgumentException ex) {
+                ex.printStackTrace();
+            } catch (InvocationTargetException ex) {
+                ex.printStackTrace();
+            } catch (IllegalAccessException ex) {
+                ex.printStackTrace();
+            }
+        }
+        return null;
+    }
+    /**
+     * Creates a new instance of RecordReader.
+     * @param in Stream from which to deserialize a record
+     * @param format Deserialization format ("binary", "xml", or "csv")
+     */
+    public RecordReader(InputStream in, String format)
+    throws IOException {
+        archive = createArchive(in, format);
+    }
+    
+    /**
+     * Deserialize a record
+     * @param r Record to be deserialized
+     */
+    public void read(Record r) throws IOException {
+        r.deserialize(archive, "");
+    }
+    
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/RecordWriter.java Wed May  3 19:04:01 2006
@@ -0,0 +1,115 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.DataOutputStream;
+import java.io.InputStream;
+import java.io.UnsupportedEncodingException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import javax.xml.parsers.ParserConfigurationException;
+import org.xml.sax.SAXException;
+
+/**
+ * Front-end for serializers. Also serves as a factory for serializers.
+ *
+ * @author milindb
+ */
+public class RecordWriter {
+    
+    private OutputArchive archive;
+    
+    static private OutputArchive getBinaryArchive(OutputStream out) {
+        return new BinaryOutputArchive(new DataOutputStream(out));
+    }
+    
+    static private OutputArchive getCsvArchive(OutputStream out)
+    throws IOException {
+        try {
+            return new CsvOutputArchive(out);
+        } catch (UnsupportedEncodingException ex) {
+            throw new IOException("Unsupported encoding UTF-8");
+        }
+    }
+    
+    static private OutputArchive getXmlArchive(OutputStream out)
+    throws IOException {
+        return new XmlOutputArchive(out);
+    }
+
+    static HashMap constructFactory() {
+        HashMap factory = new HashMap();
+        Class[] params = { OutputStream.class };
+        try {
+            factory.put("binary",
+                    BinaryOutputArchive.class.getDeclaredMethod(
+                        "getArchive", params));
+            factory.put("csv",
+                    CsvOutputArchive.class.getDeclaredMethod(
+                        "getArchive", params));
+            factory.put("xml",
+                    XmlOutputArchive.class.getDeclaredMethod(
+                        "getArchive", params));
+        } catch (SecurityException ex) {
+            ex.printStackTrace();
+        } catch (NoSuchMethodException ex) {
+            ex.printStackTrace();
+        }
+        return factory;
+    }
+    
+    static private HashMap archiveFactory = constructFactory();
+    
+    static private OutputArchive createArchive(OutputStream out,
+            String format)
+            throws IOException {
+        Method factory = (Method) archiveFactory.get(format);
+        if (factory != null) {
+            Object[] params = { out };
+            try {
+                return (OutputArchive) factory.invoke(null, params);
+            } catch (IllegalArgumentException ex) {
+                ex.printStackTrace();
+            } catch (InvocationTargetException ex) {
+                ex.printStackTrace();
+            } catch (IllegalAccessException ex) {
+                ex.printStackTrace();
+            }
+        }
+        return null;
+    }
+    /**
+     * Creates a new instance of RecordWriter
+     * @param out Output stream where the records will be serialized
+     * @param format Serialization format ("binary", "xml", or "csv")
+     */
+    public RecordWriter(OutputStream out, String format)
+    throws IOException {
+        archive = createArchive(out, format);
+    }
+    
+    /**
+     * Serialize a record
+     * @param r record to be serialized
+     */
+    public void write(Record r) throws IOException {
+        r.serialize(archive, "");
+    }
+}
\ No newline at end of file

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/Utils.java Wed May  3 19:04:01 2006
@@ -0,0 +1,366 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+
+/**
+ * Various utility functions for Hadooop record I/O runtime.
+ * @author milindb@yahoo-inc.com
+ */
+public class Utils {
+    
+    /** Cannot create a new instance of Utils */
+    private Utils() {
+    }
+   
+    /**
+     * Serializes an integer to a binary stream with zero-compressed encoding.
+     * For -120 <= i <= 127, only one byte is used with the actual value.
+     * For other values of i, the first byte value indicates whether the
+     * integer is positive or negative, and the number of bytes that follow.
+     * If the first byte value v is between -121 and -124, the following integer
+     * is positive, with number of bytes that follow are -(v+120).
+     * If the first byte value v is between -125 and -128, the following integer
+     * is negative, with number of bytes that follow are -(v+124). Bytes are
+     * stored in the high-non-zero-byte-first order.
+     *
+     * @param stream Binary output stream
+     * @param i Integer to be serialized
+     * @throws java.io.IOException 
+     */
+    static void writeInt(DataOutput stream, int i) throws IOException {
+        if (i >= -120 && i <= 127) {
+            stream.writeByte((byte)i);
+            return;
+        }
+        
+        int len = -120;
+        if (i < 0) {
+            i &= 0x7FFFFFFF; // reset the sign bit
+            len = -124;
+        }
+        
+        int tmp = i;
+        while (tmp != 0) {
+            tmp = tmp >> 8;
+            len--;
+        }
+        
+        stream.writeByte((byte)len);
+        
+        len = (len < -124) ? -(len + 124) : -(len+120);
+        
+        for (int idx = len; idx != 0; idx--) {
+            int shiftbits = (idx - 1) * 8;
+            int mask = 0xFF << shiftbits;
+            stream.writeByte((byte)((i & mask) >> shiftbits));
+        }
+    }
+    
+    /**
+     * Serializes a long to a binary stream with zero-compressed encoding.
+     * For -112 <= i <= 127, only one byte is used with the actual value.
+     * For other values of i, the first byte value indicates whether the
+     * long is positive or negative, and the number of bytes that follow.
+     * If the first byte value v is between -113 and -120, the following long
+     * is positive, with number of bytes that follow are -(v+112).
+     * If the first byte value v is between -121 and -128, the following long
+     * is negative, with number of bytes that follow are -(v+120). Bytes are
+     * stored in the high-non-zero-byte-first order.
+     * 
+     * @param stream Binary output stream
+     * @param i Long to be serialized
+     * @throws java.io.IOException 
+     */
+    static void writeLong(DataOutput stream, long i) throws IOException {
+        if (i >= -112 && i <= 127) {
+            stream.writeByte((byte)i);
+            return;
+        }
+        
+        int len = -112;
+        if (i < 0) {
+            i &= 0x7FFFFFFFFFFFFFFFL; // reset the sign bit
+            len = -120;
+        }
+        
+        long tmp = i;
+        while (tmp != 0) {
+            tmp = tmp >> 8;
+            len--;
+        }
+        
+        stream.writeByte((byte)len);
+        
+        len = (len < -120) ? -(len + 120) : -(len + 112);
+        
+        for (int idx = len; idx != 0; idx--) {
+            int shiftbits = (idx - 1) * 8;
+            long mask = 0xFFL << shiftbits;
+            stream.writeByte((byte)((i & mask) >> shiftbits));
+        }
+    }
+    
+    /**
+     * Reads a zero-compressed encoded integer from input stream and returns it.
+     * @param stream Binary input stream
+     * @throws java.io.IOException 
+     * @return deserialized integer from stream.
+     */
+    static int readInt(DataInput stream) throws IOException {
+        int len = stream.readByte();
+        if (len >= -120) {
+            return len;
+        }
+        len = (len < -124) ? -(len + 124) : -(len + 120);
+        byte[] barr = new byte[len];
+        stream.readFully(barr);
+        int i = 0;
+        for (int idx = 0; idx < len; idx++) {
+            i = i << 8;
+            i = i | (barr[idx] & 0xFF);
+        }
+        return i;
+    }
+    
+    /**
+     * Reads a zero-compressed encoded long from input stream and returns it.
+     * @param stream Binary input stream
+     * @throws java.io.IOException 
+     * @return deserialized long from stream.
+     */
+    static long readLong(DataInput stream) throws IOException {
+        int len = stream.readByte();
+        if (len >= -112) {
+            return len;
+        }
+        len = (len < -120) ? -(len + 120) : -(len + 112);
+        byte[] barr = new byte[len];
+        stream.readFully(barr);
+        long i = 0;
+        for (int idx = 0; idx < len; idx++) {
+            i = i << 8;
+            i = i | (barr[idx] & 0xFF);
+        }
+        return i;
+    }
+    
+    /**
+     * equals function that actually compares two buffers.
+     *
+     * @param one First buffer
+     * @param two Second buffer
+     * @return true if one and two contain exactly the same content, else false.
+     */
+    public static boolean bufEquals(ByteArrayOutputStream one,
+            ByteArrayOutputStream two) {
+        if (one == two) {
+            return true;
+        }
+        byte[] onearray = one.toByteArray();
+        byte[] twoarray = two.toByteArray();
+        boolean ret = (onearray.length == twoarray.length);
+        if (!ret) {
+            return ret;
+        }
+        for (int idx = 0; idx < onearray.length; idx++) {
+            if (onearray[idx] != twoarray[idx]) {
+                return false;
+            }
+        }
+        return true;
+    }
+    
+    /**
+     * 
+     * @param s 
+     * @return 
+     */
+    static String toXMLString(String s) {
+        String rets = "";
+        try {
+            rets = java.net.URLEncoder.encode(s, "UTF-8");
+        } catch (UnsupportedEncodingException ex) {
+            ex.printStackTrace();
+        }
+        return rets;
+    }
+    
+    /**
+     * 
+     * @param s 
+     * @return 
+     */
+    static String fromXMLString(String s) {
+        String rets = "";
+        try {
+            rets = java.net.URLDecoder.decode(s, "UTF-8");
+        } catch (UnsupportedEncodingException ex) {
+            ex.printStackTrace();
+        }
+        return rets;
+    }
+    
+    /**
+     * 
+     * @param s 
+     * @return 
+     */
+    static String toCSVString(String s) {
+        StringBuffer sb = new StringBuffer(s.length()+1);
+        sb.append('\'');
+        int len = s.length();
+        for (int i = 0; i < len; i++) {
+            char c = s.charAt(i);
+            switch(c) {
+                case '\0':
+                    sb.append("%00");
+                    break;
+                case '\n':
+                    sb.append("%0A");
+                    break;
+                case '\r':
+                    sb.append("%0D");
+                    break;
+                case ',':
+                    sb.append("%2C");
+                    break;
+                case '}':
+                    sb.append("%7D");
+                    break;
+                case '%':
+                    sb.append("%25");
+                    break;
+                default:
+                    sb.append(c);
+            }
+        }
+        return sb.toString();
+    }
+    
+    /**
+     * 
+     * @param s 
+     * @throws java.io.IOException 
+     * @return 
+     */
+    static String fromCSVString(String s) throws IOException {
+        if (s.charAt(0) != '\'') {
+            throw new IOException("Error deserializing string.");
+        }
+        int len = s.length();
+        StringBuffer sb = new StringBuffer(len-1);
+        for (int i = 1; i < len; i++) {
+            char c = s.charAt(i);
+            if (c == '%') {
+                char ch1 = s.charAt(i+1);
+                char ch2 = s.charAt(i+2);
+                i += 2;
+                if (ch1 == '0' && ch2 == '0') { sb.append('\0'); }
+                else if (ch1 == '0' && ch2 == 'A') { sb.append('\n'); }
+                else if (ch1 == '0' && ch2 == 'D') { sb.append('\r'); }
+                else if (ch1 == '2' && ch2 == 'C') { sb.append(','); }
+                else if (ch1 == '7' && ch2 == 'D') { sb.append('}'); }
+                else if (ch1 == '2' && ch2 == '5') { sb.append('%'); }
+                else {throw new IOException("Error deserializing string.");}
+            } else {
+                sb.append(c);
+            }
+        }
+        return sb.toString();
+    }
+    
+    /**
+     * 
+     * @param s 
+     * @return 
+     */
+    static String toXMLBuffer(ByteArrayOutputStream s) {
+        byte[] barr = s.toByteArray();
+        StringBuffer sb = new StringBuffer(2*barr.length);
+        for (int idx = 0; idx < barr.length; idx++) {
+            sb.append(Integer.toHexString((int)barr[idx]));
+        }
+        return sb.toString();
+    }
+    
+    /**
+     * 
+     * @param s 
+     * @throws java.io.IOException 
+     * @return 
+     */
+    static ByteArrayOutputStream fromXMLBuffer(String s)
+    throws IOException {
+        ByteArrayOutputStream stream =  new ByteArrayOutputStream();
+        if (s.length() == 0) { return stream; }
+        int blen = s.length()/2;
+        byte[] barr = new byte[blen];
+        for (int idx = 0; idx < blen; idx++) {
+            char c1 = s.charAt(2*idx);
+            char c2 = s.charAt(2*idx+1);
+            barr[idx] = Byte.parseByte(""+c1+c2, 16);
+        }
+        stream.write(barr);
+        return stream;
+    }
+    
+    /**
+     * 
+     * @param buf 
+     * @return 
+     */
+    static String toCSVBuffer(ByteArrayOutputStream buf) {
+        byte[] barr = buf.toByteArray();
+        StringBuffer sb = new StringBuffer(barr.length+1);
+        sb.append('#');
+        for(int idx = 0; idx < barr.length; idx++) {
+            sb.append(Integer.toHexString((int)barr[idx]));
+        }
+        return sb.toString();
+    }
+    
+    /**
+     * Converts a CSV-serialized representation of buffer to a new
+     * ByteArrayOutputStream.
+     * @param s CSV-serialized representation of buffer
+     * @throws java.io.IOException 
+     * @return Deserialized ByteArrayOutputStream
+     */
+    static ByteArrayOutputStream fromCSVBuffer(String s)
+    throws IOException {
+        if (s.charAt(0) != '#') {
+            throw new IOException("Error deserializing buffer.");
+        }
+        ByteArrayOutputStream stream =  new ByteArrayOutputStream();
+        if (s.length() == 1) { return stream; }
+        int blen = (s.length()-1)/2;
+        byte[] barr = new byte[blen];
+        for (int idx = 0; idx < blen; idx++) {
+            char c1 = s.charAt(2*idx+1);
+            char c2 = s.charAt(2*idx+2);
+            barr[idx] = Byte.parseByte(""+c1+c2, 16);
+        }
+        stream.write(barr);
+        return stream;
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlInputArchive.java Wed May  3 19:04:01 2006
@@ -0,0 +1,250 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.InputStream;
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.xml.sax.*;
+import org.xml.sax.helpers.DefaultHandler;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+/**
+ *
+ * @author milindb
+ */
+class XmlInputArchive implements InputArchive {
+    
+    static private class Value {
+        private String type;
+        private StringBuffer sb;
+        
+        public Value(String t) {
+            type = t;
+            sb = new StringBuffer();
+        }
+        public void addChars(char[] buf, int offset, int len) {
+            sb.append(buf, offset, len);
+        }
+        public String getValue() { return sb.toString(); }
+        public String getType() { return type; }
+    }
+    
+    private class XMLParser extends DefaultHandler {
+        private boolean charsValid = false;
+        
+        private ArrayList valList;
+        
+        private XMLParser(ArrayList vlist) {
+            valList = vlist;
+        }
+        
+        public void startDocument() throws SAXException {}
+        
+        public void endDocument() throws SAXException {}
+        
+        public void startElement(String ns,
+                String sname,
+                String qname,
+                Attributes attrs) throws SAXException {
+            charsValid = false;
+            if ("boolean".equals(qname) ||
+                    "i4".equals(qname) ||
+                    "int".equals(qname) ||
+                    "string".equals(qname) ||
+                    "double".equals(qname) ||
+                    "ex:i1".equals(qname) ||
+                    "ex:i8".equals(qname) ||
+                    "ex:float".equals(qname)) {
+                charsValid = true;
+                valList.add(new Value(qname));
+            } else if ("struct".equals(qname) ||
+                "array".equals(qname)) {
+                valList.add(new Value(qname));
+            }
+        }
+        
+        public void endElement(String ns,
+                String sname,
+                String qname) throws SAXException {
+            charsValid = false;
+            if ("struct".equals(qname) ||
+                    "array".equals(qname)) {
+                valList.add(new Value("/"+qname));
+            }
+        }
+        
+        public void characters(char buf[], int offset, int len)
+        throws SAXException {
+            if (charsValid) {
+                Value v = (Value) valList.get(valList.size()-1);
+                v.addChars(buf, offset,len);
+            }
+        }
+        
+    }
+    
+    private class XmlIndex implements Index {
+        public boolean done() {
+            Value v = (Value) valList.get(vIdx);
+            if ("/array".equals(v.getType())) {
+                valList.set(vIdx, null);
+                vIdx++;
+                return true;
+            } else {
+                return false;
+            }
+        }
+        public void incr() {}
+    }
+    
+    private ArrayList valList;
+    private int vLen;
+    private int vIdx;
+    
+    private Value next() throws IOException {
+        if (vIdx < vLen) {
+            Value v = (Value) valList.get(vIdx);
+            valList.set(vIdx, null);
+            vIdx++;
+            return v;
+        } else {
+            throw new IOException("Error in deserialization.");
+        }
+    }
+    
+    static XmlInputArchive getArchive(InputStream strm)
+    throws ParserConfigurationException, SAXException, IOException {
+        return new XmlInputArchive(strm);
+    }
+    
+    /** Creates a new instance of BinaryInputArchive */
+    public XmlInputArchive(InputStream in)
+    throws ParserConfigurationException, SAXException, IOException {
+        valList = new ArrayList();
+        DefaultHandler handler = new XMLParser(valList);
+        SAXParserFactory factory = SAXParserFactory.newInstance();
+        SAXParser parser = factory.newSAXParser();
+        parser.parse(in, handler);
+        vLen = valList.size();
+        vIdx = 0;
+    }
+    
+    public byte readByte(String tag) throws IOException {
+        Value v = next();
+        if (!"ex:i1".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Byte.parseByte(v.getValue());
+    }
+    
+    public boolean readBool(String tag) throws IOException {
+        Value v = next();
+        if (!"boolean".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return "1".equals(v.getValue());
+    }
+    
+    public int readInt(String tag) throws IOException {
+        Value v = next();
+        if (!"i4".equals(v.getType()) &&
+                !"int".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Integer.parseInt(v.getValue());
+    }
+    
+    public long readLong(String tag) throws IOException {
+        Value v = next();
+        if (!"ex:i8".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Long.parseLong(v.getValue());
+    }
+    
+    public float readFloat(String tag) throws IOException {
+        Value v = next();
+        if (!"ex:float".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Float.parseFloat(v.getValue());
+    }
+    
+    public double readDouble(String tag) throws IOException {
+        Value v = next();
+        if (!"double".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Double.parseDouble(v.getValue());
+    }
+    
+    public String readString(String tag) throws IOException {
+        Value v = next();
+        if (!"string".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Utils.fromXMLString(v.getValue());
+    }
+    
+    public ByteArrayOutputStream readBuffer(String tag) throws IOException {
+        Value v = next();
+        if (!"string".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return Utils.fromXMLBuffer(v.getValue());
+    }
+    
+    public void readRecord(Record r, String tag) throws IOException {
+        r.deserialize(this, tag);
+    }
+    
+    public void startRecord(String tag) throws IOException {
+        Value v = next();
+        if (!"struct".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+    }
+    
+    public void endRecord(String tag) throws IOException {
+        Value v = next();
+        if (!"/struct".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+    }
+    
+    public Index startVector(String tag) throws IOException {
+        Value v = next();
+        if (!"array".equals(v.getType())) {
+            throw new IOException("Error deserializing "+tag+".");
+        }
+        return new XmlIndex();
+    }
+    
+    public void endVector(String tag) throws IOException {}
+    
+    public Index startMap(String tag) throws IOException {
+        return startVector(tag);
+    }
+    
+    public void endMap(String tag) throws IOException { endVector(tag); }
+
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/XmlOutputArchive.java Wed May  3 19:04:01 2006
@@ -0,0 +1,251 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record;
+
+import java.io.IOException;
+import java.io.ByteArrayOutputStream;
+import java.util.TreeMap;
+import java.util.ArrayList;
+import java.io.PrintStream;
+import java.io.OutputStream;
+import java.util.Stack;
+
+/**
+ *
+ * @author milindb
+ */
+class XmlOutputArchive implements OutputArchive {
+
+    private PrintStream stream;
+    
+    private int indent = 0;
+    
+    private Stack compoundStack;
+    
+    private void putIndent() {
+        StringBuffer sb = new StringBuffer("");
+        for (int idx = 0; idx < indent; idx++) {
+            sb.append("  ");
+        }
+        stream.print(sb.toString());
+    }
+    
+    private void addIndent() {
+        indent++;
+    }
+    
+    private void closeIndent() {
+        indent--;
+    }
+    
+    private void printBeginEnvelope(String tag) {
+        if (!compoundStack.empty()) {
+            String s = (String) compoundStack.peek();
+            if ("struct".equals(s)) {
+                putIndent();
+                stream.print("<member>\n");
+                addIndent();
+                putIndent();
+                stream.print("<name>"+tag+"</name>\n");
+                putIndent();
+                stream.print("<value>");
+            } else if ("vector".equals(s)) {
+                stream.print("<value>");
+            } else if ("map".equals(s)) {
+                stream.print("<value>");
+            }
+        } else {
+            stream.print("<value>");
+        }
+    }
+    
+    private void printEndEnvelope(String tag) {
+        if (!compoundStack.empty()) {
+            String s = (String) compoundStack.peek();
+            if ("struct".equals(s)) {
+                stream.print("</value>\n");
+                closeIndent();
+                putIndent();
+                stream.print("</member>\n");
+            } else if ("vector".equals(s)) {
+                stream.print("</value>\n");
+            } else if ("map".equals(s)) {
+                stream.print("</value>\n");
+            }
+        } else {
+            stream.print("</value>\n");
+        }
+    }
+    
+    private void insideVector(String tag) {
+        printBeginEnvelope(tag);
+        compoundStack.push("vector");
+    }
+    
+    private void outsideVector(String tag) throws IOException {
+        String s = (String) compoundStack.pop();
+        if (!"vector".equals(s)) {
+            throw new IOException("Error serializing vector.");
+        }
+        printEndEnvelope(tag);
+    }
+    
+    private void insideMap(String tag) {
+        printBeginEnvelope(tag);
+        compoundStack.push("map");
+    }
+    
+    private void outsideMap(String tag) throws IOException {
+        String s = (String) compoundStack.pop();
+        if (!"map".equals(s)) {
+            throw new IOException("Error serializing map.");
+        }
+        printEndEnvelope(tag);
+    }
+    
+    private void insideRecord(String tag) {
+        printBeginEnvelope(tag);
+        compoundStack.push("struct");
+    }
+    
+    private void outsideRecord(String tag) throws IOException {
+        String s = (String) compoundStack.pop();
+        if (!"struct".equals(s)) {
+            throw new IOException("Error serializing record.");
+        }
+        printEndEnvelope(tag);
+    }
+    
+    static XmlOutputArchive getArchive(OutputStream strm) {
+        return new XmlOutputArchive(strm);
+    }
+    
+    /** Creates a new instance of XmlOutputArchive */
+    public XmlOutputArchive(OutputStream out) {
+        stream = new PrintStream(out);
+        compoundStack = new Stack();
+    }
+    
+    public void writeByte(byte b, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<ex:i1>");
+        stream.print(Byte.toString(b));
+        stream.print("</ex:i1>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeBool(boolean b, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<boolean>");
+        stream.print(b ? "1" : "0");
+        stream.print("</boolean>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeInt(int i, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<i4>");
+        stream.print(Integer.toString(i));
+        stream.print("</i4>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeLong(long l, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<ex:i8>");
+        stream.print(Long.toString(l));
+        stream.print("</ex:i8>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeFloat(float f, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<ex:float>");
+        stream.print(Float.toString(f));
+        stream.print("</ex:float>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeDouble(double d, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<double>");
+        stream.print(Double.toString(d));
+        stream.print("</double>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeString(String s, String tag) throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<string>");
+        stream.print(Utils.toXMLString(s));
+        stream.print("</string>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeBuffer(ByteArrayOutputStream buf, String tag)
+    throws IOException {
+        printBeginEnvelope(tag);
+        stream.print("<string>");
+        stream.print(Utils.toXMLBuffer(buf));
+        stream.print("</string>");
+        printEndEnvelope(tag);
+    }
+    
+    public void writeRecord(Record r, String tag) throws IOException {
+        r.serialize(this, tag);
+    }
+    
+    public void startRecord(Record r, String tag) throws IOException {
+        insideRecord(tag);
+        stream.print("<struct>\n");
+        addIndent();
+    }
+    
+    public void endRecord(Record r, String tag) throws IOException {
+        closeIndent();
+        putIndent();
+        stream.print("</struct>");
+        outsideRecord(tag);
+    }
+    
+    public void startVector(ArrayList v, String tag) throws IOException {
+        insideVector(tag);
+        stream.print("<array>\n");
+        addIndent();
+    }
+    
+    public void endVector(ArrayList v, String tag) throws IOException {
+        closeIndent();
+        putIndent();
+        stream.print("</array>");
+        outsideVector(tag);
+    }
+    
+    public void startMap(TreeMap v, String tag) throws IOException {
+        insideMap(tag);
+        stream.print("<array>\n");
+        addIndent();
+    }
+    
+    public void endMap(TreeMap v, String tag) throws IOException {
+        closeIndent();
+        putIndent();
+        stream.print("</array>");
+        outsideMap(tag);
+    }
+
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java Wed May  3 19:04:01 2006
@@ -0,0 +1,64 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+import java.util.ArrayList;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ *
+ * @author milindb
+ */
+class CppGenerator {
+    private String mName;
+    private ArrayList mInclFiles;
+    private ArrayList mRecList;
+    
+    /** Creates a new instance of CppGenerator */
+    public CppGenerator(String name, ArrayList ilist, ArrayList rlist) {
+        mName = name;
+        mInclFiles = ilist;
+        mRecList = rlist;
+    }
+    
+    public void genCode() throws IOException {
+        FileWriter cc = new FileWriter(mName+".cc");
+        FileWriter hh = new FileWriter(mName+".hh");
+        hh.write("#ifndef __"+mName.toUpperCase().replace('.','_')+"__\n");
+        hh.write("#define __"+mName.toUpperCase().replace('.','_')+"__\n");
+        
+        hh.write("#include \"recordio.hh\"\n");
+        for (Iterator i = mInclFiles.iterator(); i.hasNext();) {
+            JFile f = (JFile) i.next();
+            hh.write("#include \""+f.getName()+".hh\"\n");
+        }
+        cc.write("#include \""+mName+".hh\"\n");
+        
+        for (Iterator i = mRecList.iterator(); i.hasNext();) {
+            JRecord jr = (JRecord) i.next();
+            jr.genCppCode(hh, cc);
+        }
+        
+        hh.write("#endif //"+mName.toUpperCase().replace('.','_')+"__\n");
+        
+        hh.close();
+        cc.close();
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java Wed May  3 19:04:01 2006
@@ -0,0 +1,41 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+/**
+ *
+ * @author milindb
+ */
+public class JBoolean extends JType {
+    
+    /** Creates a new instance of JBoolean */
+    public JBoolean() {
+        super("bool", "boolean", "Bool", "Boolean", "toBoolean");
+    }
+    
+    public String getSignature() {
+        return "z";
+    }
+    
+    public String genJavaCompareTo(String fname) {
+        return "    ret = ("+fname+" == peer."+fname+")? 0 : ("+fname+"?1:-1);\n";
+    }
+    
+    public String genJavaHashCode(String fname) {
+        return "     ret = ("+fname+")?0:1;\n";
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java Wed May  3 19:04:01 2006
@@ -0,0 +1,67 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+/**
+ *
+ * @author milindb
+ */
+public class JBuffer extends JCompType {
+    
+    /** Creates a new instance of JBuffer */
+    public JBuffer() {
+        super(" ::std::string", "java.io.ByteArrayOutputStream", "Buffer", "java.io.ByteArrayOutputStream");
+    }
+    
+    public String genCppGetSet(String fname, int fIdx) {
+        String cgetFunc = "  virtual const "+getCppType()+"& get"+fname+"() const {\n";
+        cgetFunc += "    return m"+fname+";\n";
+        cgetFunc += "  }\n";
+        String getFunc = "  virtual "+getCppType()+"& get"+fname+"() {\n";
+        getFunc += "    bs_.set("+fIdx+");return m"+fname+";\n";
+        getFunc += "  }\n";
+        return cgetFunc + getFunc;
+    }
+    
+    public String getSignature() {
+        return "B";
+    }
+    
+    public String genJavaReadWrapper(String fname, String tag, boolean decl) {
+        String ret = "";
+        if (decl) {
+            ret = "    java.io.ByteArrayOutputStream "+fname+";\n";
+        }
+        return ret + "        java.io.ByteArrayOutputStream "+fname+"=a_.readBuffer(\""+tag+"\");\n";
+    }
+    
+    public String genJavaWriteWrapper(String fname, String tag) {
+        return "        a_.writeBuffer("+fname+"\""+tag+"\");\n";
+    }
+    
+    public String genJavaCompareTo(String fname) {
+        return "";
+    }
+    
+    public String genJavaEquals(String fname, String peer) {
+        return "    ret = org.apache.hadoop.record.Utils.bufEquals("+fname+","+peer+");\n";
+    }
+    
+    public String genJavaHashCode(String fname) {
+        return "    ret = "+fname+".toString().hashCode();\n";
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java Wed May  3 19:04:01 2006
@@ -0,0 +1,33 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+/**
+ *
+ * @author milindb
+ */
+public class JByte extends JType {
+    
+    /** Creates a new instance of JByte */
+    public JByte() {
+        super("int8_t", "byte", "Byte", "Byte", "toByte");
+    }
+    
+    public String getSignature() {
+        return "b";
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java Wed May  3 19:04:01 2006
@@ -0,0 +1,51 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+/**
+ *
+ * @author milindb
+ */
+abstract class JCompType extends JType {
+    
+    /** Creates a new instance of JCompType */
+    public JCompType(String cppType, String javaType, String suffix, String wrapper) {
+        super(cppType, javaType, suffix, wrapper, null);
+    }
+    
+    public String genCppGetSet(String fname, int fIdx) {
+        String cgetFunc = "  virtual const "+getCppType()+"& get"+fname+"() const {\n";
+        cgetFunc += "    return m"+fname+";\n";
+        cgetFunc += "  }\n";
+        String getFunc = "  virtual "+getCppType()+"& get"+fname+"() {\n";
+        getFunc += "    bs_.set("+fIdx+");return m"+fname+";\n";
+        getFunc += "  }\n";
+        return cgetFunc + getFunc;
+    }
+    
+    public String genJavaCompareTo(String fname) {
+        return "    ret = "+fname+".compareTo(peer."+fname+");\n";
+    }
+    
+    public String genJavaEquals(String fname, String peer) {
+        return "    ret = "+fname+".equals("+peer+");\n";
+    }
+    
+    public String genJavaHashCode(String fname) {
+        return "    ret = "+fname+".hashCode();\n";
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java Wed May  3 19:04:01 2006
@@ -0,0 +1,38 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+/**
+ *
+ * @author milindb
+ */
+public class JDouble extends JType {
+    
+    /** Creates a new instance of JDouble */
+    public JDouble() {
+        super("double", "double", "Double", "Double", "toDouble");
+    }
+    
+    public String getSignature() {
+        return "d";
+    }
+    
+    public String genJavaHashCode(String fname) {
+        String tmp = "Double.doubleToLongBits("+fname+")";
+        return "    ret = (int)("+tmp+"^("+tmp+">>>32));\n";
+    }
+}

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java
URL: http://svn.apache.org/viewcvs/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java?rev=399509&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JField.java Wed May  3 19:04:01 2006
@@ -0,0 +1,93 @@
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+/**
+ *
+ * @author milindb
+ */
+public class JField {
+    private JType mType;
+    private String mName;
+    /**
+     * Creates a new instance of JField
+     */
+    public JField(JType type, String name) {
+        mType = type;
+        mName = name;
+    }
+    
+    public String getSignature() {
+        return mType.getSignature();
+    }
+    
+    public String genCppDecl() {
+        return mType.genCppDecl(mName);
+    }
+    
+    public String genJavaDecl() {
+        return mType.genJavaDecl(mName);
+    }
+    
+    public String genJavaConstructorParam(int fIdx) {
+        return mType.genJavaConstructorParam(fIdx);
+    }
+    
+    public String getName() {
+        return "m"+mName;
+    }
+    
+    public String getTag() {
+        return mName;
+    }
+    
+    public JType getType() {
+        return mType;
+    }
+    
+    public String genCppGetSet(int fIdx) {
+        return mType.genCppGetSet(mName, fIdx);
+    }
+    
+    public String genJavaGetSet(int fIdx) {
+        return mType.genJavaGetSet(mName, fIdx);
+    }
+    
+    public String genJavaWriteMethodName() {
+        return mType.genJavaWriteMethod(getName(), getTag());
+    }
+    
+    public String genJavaReadMethodName() {
+        return mType.genJavaReadMethod(getName(), getTag());
+    }
+    
+    public String genJavaCompareTo() {
+        return mType.genJavaCompareTo(getName());
+    }
+    
+    public String genJavaEquals() {
+        return mType.genJavaEquals(getName(), "peer."+getName());
+    }
+    
+    public String genJavaHashCode() {
+        return mType.genJavaHashCode(getName());
+    }
+
+    public String genJavaConstructorSet(int fIdx) {
+        return mType.genJavaConstructorSet(mName, fIdx);
+    }
+}



Mime
View raw message