hadoop-common-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject svn commit: r612774 [1/2] - in /lucene/hadoop/trunk: ./ src/c++/librecordio/ src/c++/librecordio/test/ src/java/org/apache/hadoop/record/compiler/ src/java/org/apache/hadoop/record/meta/ src/test/ddl/ src/test/org/apache/hadoop/record/
Date Thu, 17 Jan 2008 09:43:42 GMT
Author: ddas
Date: Thu Jan 17 01:43:33 2008
New Revision: 612774

URL: http://svn.apache.org/viewvc?rev=612774&view=rev
Log:
HADOOP-1883. Fixed some commit problems.

Added:
    lucene/hadoop/trunk/src/c++/librecordio/fieldTypeInfo.cc
    lucene/hadoop/trunk/src/c++/librecordio/fieldTypeInfo.hh
    lucene/hadoop/trunk/src/c++/librecordio/recordTypeInfo.cc
    lucene/hadoop/trunk/src/c++/librecordio/recordTypeInfo.hh
    lucene/hadoop/trunk/src/c++/librecordio/typeIDs.cc
    lucene/hadoop/trunk/src/c++/librecordio/typeIDs.hh
    lucene/hadoop/trunk/src/c++/librecordio/typeInfo.cc
    lucene/hadoop/trunk/src/c++/librecordio/typeInfo.hh
    lucene/hadoop/trunk/src/c++/librecordio/utils.cc
    lucene/hadoop/trunk/src/c++/librecordio/utils.hh
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/Consts.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/meta/
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/meta/FieldTypeInfo.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/meta/MapTypeID.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/meta/RecordTypeInfo.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/meta/StructTypeID.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/meta/TypeID.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/meta/Utils.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/meta/VectorTypeID.java
    lucene/hadoop/trunk/src/test/org/apache/hadoop/record/TestRecordVersioning.java
Modified:
    lucene/hadoop/trunk/CHANGES.txt
    lucene/hadoop/trunk/src/c++/librecordio/Makefile
    lucene/hadoop/trunk/src/c++/librecordio/recordio.cc
    lucene/hadoop/trunk/src/c++/librecordio/recordio.hh
    lucene/hadoop/trunk/src/c++/librecordio/test/test.cc
    lucene/hadoop/trunk/src/c++/librecordio/test/test.jr
    lucene/hadoop/trunk/src/c++/librecordio/test/testFromJava.cc
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFloat.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JInt.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JLong.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JRecord.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JString.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JType.java
    lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JVector.java
    lucene/hadoop/trunk/src/test/ddl/test.jr

Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Jan 17 01:43:33 2008
@@ -90,6 +90,8 @@
 
     HADOOP-2532.  Add to MapFile a getClosest method that returns the key
     that comes just before if the key is not present.  (stack via tomwhite)
+   
+    HADOOP-1883. Add versioning to Record I/O. (Vivek Ratan via ddas)
 
   IMPROVEMENTS
 

Modified: lucene/hadoop/trunk/src/c++/librecordio/Makefile
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/Makefile?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/Makefile (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/Makefile Thu Jan 17 01:43:33 2008
@@ -18,15 +18,17 @@
 
 all: librecordio.a test
 
-librecordio.a: recordio.o filestream.o binarchive.o csvarchive.o xmlarchive.o exception.o
-	ar cru librecordio.a recordio.o filestream.o binarchive.o csvarchive.o xmlarchive.o exception.o
+librecordio.a: recordio.o filestream.o binarchive.o csvarchive.o xmlarchive.o \
+	exception.o typeIDs.o fieldTypeInfo.o recordTypeInfo.o utils.o
+	ar cru librecordio.a recordio.o filestream.o binarchive.o csvarchive.o xmlarchive.o \
+	exception.o typeIDs.o fieldTypeInfo.o recordTypeInfo.o utils.o
 
 recordio.o: recordio.cc recordio.hh archive.hh
 	g++ ${COPTS} -c -I${XERCESCROOT}/include -o recordio.o recordio.cc
-	
+
 filestream.o: filestream.cc recordio.hh filestream.hh
 	g++ ${COPTS} -c -o filestream.o filestream.cc
-	
+
 binarchive.o: binarchive.cc recordio.hh binarchive.hh archive.hh
 	g++ ${COPTS} -c -o binarchive.o binarchive.cc
 
@@ -35,20 +37,33 @@
 
 xmlarchive.o: xmlarchive.cc recordio.hh xmlarchive.hh archive.hh
 	g++ ${COPTS} -c -I${XERCESCROOT}/include -o xmlarchive.o xmlarchive.cc
-		
+
 exception.o: exception.cc exception.hh
 	g++ ${COPTS} -c -o exception.o exception.cc
-	
+
+typeIDs.o: typeIDs.cc typeIDs.hh
+	g++ ${COPTS} -c -o typeIDs.o typeIDs.cc
+fieldTypeInfo.o: fieldTypeInfo.cc fieldTypeInfo.hh
+	g++ ${COPTS} -c -o fieldTypeInfo.o fieldTypeInfo.cc
+recordTypeInfo.o: recordTypeInfo.cc recordTypeInfo.hh
+	g++ ${COPTS} -c -o recordTypeInfo.o recordTypeInfo.cc
+utils.o: utils.cc utils.hh
+	g++ ${COPTS} -c -o utils.o utils.cc
+
 recordio.cc: recordio.hh archive.hh exception.hh
 filestream.cc: recordio.hh filestream.hh 
 binarchive.cc: recordio.hh binarchive.hh 
 csvarchive.cc: recordio.hh csvarchive.hh 
 xmlarchive.cc: recordio.hh xmlarchive.hh 
 exception.cc: exception.hh 
+typeIDs.cc: typeIDs.hh
+fieldTypeInfo.cc: fieldTypeInfo.hh
+recordTypeInfo.cc: recordTypeInfo.hh
+utils.cc: utils.hh
 
 test: librecordio.a
 	make -C test all
-	
+
 clean:
 	rm -f *~ *.o *.a
 	make -C test clean

Added: lucene/hadoop/trunk/src/c++/librecordio/fieldTypeInfo.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/fieldTypeInfo.cc?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/fieldTypeInfo.cc (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/fieldTypeInfo.cc Thu Jan 17 01:43:33 2008
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "fieldTypeInfo.hh"
+
+using namespace hadoop;
+
+FieldTypeInfo::~FieldTypeInfo()
+{
+  delete pFieldID;
+  delete pTypeID;
+}
+
+FieldTypeInfo::FieldTypeInfo(const FieldTypeInfo& ti)
+{
+  pFieldID = new std::string(*ti.pFieldID);
+  pTypeID = ti.pTypeID->clone();
+}
+
+
+void FieldTypeInfo::serialize(::hadoop::OArchive& a_, const char* tag) const
+{
+  a_.serialize(*pFieldID, tag);
+  pTypeID->serialize(a_, tag);
+}
+
+bool FieldTypeInfo::operator==(const FieldTypeInfo& peer_) const 
+{
+  // first check if fieldID matches
+  if (0 != pFieldID->compare(*(peer_.pFieldID))) {
+    return false;
+  }
+  // now see if typeID matches
+  return (*pTypeID == *(peer_.pTypeID));
+}
+
+// print out the structure. Helps in debugging.
+void FieldTypeInfo::print(int space) const
+{
+  for (int i=0; i<space; i++) {
+    printf(" ");
+  }
+  printf("FieldTypeInfo(%lx):\n", (long)this);
+  for (int i=0; i<space+2; i++) {
+    printf(" ");
+  }
+  printf("field = \"%s\"\n", pFieldID->c_str());
+  pTypeID->print(space+2);
+}

Added: lucene/hadoop/trunk/src/c++/librecordio/fieldTypeInfo.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/fieldTypeInfo.hh?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/fieldTypeInfo.hh (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/fieldTypeInfo.hh Thu Jan 17 01:43:33 2008
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef FIELDTYPEINFO_HH_
+#define FIELDTYPEINFO_HH_
+
+#include "recordio.hh"
+#include "typeIDs.hh"
+
+namespace hadoop {
+
+class TypeID;
+
+/** 
+ * Represents a type information for a field, which is made up of its 
+ * ID (name) and its type (a TypeID object).
+ */
+class FieldTypeInfo {
+  
+private: 
+  // we own memory mgmt of these vars
+  const std::string* pFieldID;
+  const TypeID* pTypeID;
+
+public: 
+  FieldTypeInfo(const std::string* pFieldID, const TypeID* pTypeID) : 
+    pFieldID(pFieldID), pTypeID(pTypeID) {}
+  FieldTypeInfo(const FieldTypeInfo& ti);
+  virtual ~FieldTypeInfo();
+
+  const TypeID* getTypeID() const {return pTypeID;}
+  const std::string* getFieldID() const {return pFieldID;}
+  void serialize(::hadoop::OArchive& a_, const char* tag) const;
+  bool operator==(const FieldTypeInfo& peer_) const;
+  FieldTypeInfo* clone() const {return new FieldTypeInfo(*this);}
+
+  void print(int space=0) const;
+
+};
+
+}
+
+#endif // FIELDTYPEINFO_HH_
+

Added: lucene/hadoop/trunk/src/c++/librecordio/recordTypeInfo.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/recordTypeInfo.cc?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/recordTypeInfo.cc (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/recordTypeInfo.cc Thu Jan 17 01:43:33 2008
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "recordTypeInfo.hh"
+
+using namespace hadoop;
+
+RecordTypeInfo::RecordTypeInfo() 
+{
+  pStid = new StructTypeID();
+}
+
+RecordTypeInfo::RecordTypeInfo(const char *pName): name(pName)
+{
+  pStid = new StructTypeID();
+}
+
+
+/*RecordTypeInfo::RecordTypeInfo(const RecordTypeInfo& rti): name(rti.name)
+{
+  // clone the typeinfos from rti and add them
+  for (unsigned int i=0; i<rti.typeInfos.size(); i++) {
+    typeInfos.push_back(rti.typeInfos[i]->clone());
+  }
+  // clone the map
+  for (std::map<std::string, RecordTypeInfo*>::const_iterator iter=rti.structRTIs.begin(); 
+       iter!=rti.structRTIs.end(); ++iter) {
+    structRTIs[iter->first] = iter->second->clone();
+  }
+}*/
+
+
+RecordTypeInfo::~RecordTypeInfo()
+{
+  if (NULL != pStid) 
+    delete pStid;
+
+  /*for (unsigned int i=0; i<typeInfos.size(); i++) {
+    delete typeInfos[i];
+  }
+  typeInfos.clear();
+  for (std::map<std::string, RecordTypeInfo*>::const_iterator iter=structRTIs.begin(); 
+       iter!=structRTIs.end(); ++iter) {
+    // delete the RTI objects
+    delete iter->second;
+  }
+  structRTIs.clear();*/
+}
+
+void RecordTypeInfo::addField(const std::string* pFieldID, const TypeID* pTypeID)
+{
+  pStid->getFieldTypeInfos().push_back(new FieldTypeInfo(pFieldID, pTypeID));
+}
+
+void RecordTypeInfo::addAll(std::vector<FieldTypeInfo*>& vec)
+{
+  // we need to copy object clones into our own vector
+  for (unsigned int i=0; i<vec.size(); i++) {
+    pStid->getFieldTypeInfos().push_back(vec[i]->clone());
+  }
+}
+
+// make a copy of typeInfos and return it
+/*std::vector<TypeInfo*>& RecordTypeInfo::getClonedTypeInfos()
+{
+  std::vector<TypeInfo*>* pNewVec = new std::vector<TypeInfo*>();
+  for (unsigned int i=0; i<typeInfos.size(); i++) {
+    pNewVec->push_back(typeInfos[i]->clone());
+  }
+  return *pNewVec;
+} */
+
+const std::vector<FieldTypeInfo*>& RecordTypeInfo::getFieldTypeInfos() const
+{
+  return pStid->getFieldTypeInfos();
+}
+
+
+RecordTypeInfo* RecordTypeInfo::getNestedStructTypeInfo(const char *structName) const
+{
+  StructTypeID* p = pStid->findStruct(structName);
+  if (NULL == p) return NULL;
+  return new RecordTypeInfo(structName, p);
+  /*std::string s(structName);
+  std::map<std::string, RecordTypeInfo*>::const_iterator iter = structRTIs.find(s);
+  if (iter == structRTIs.end()) {
+    return NULL;
+  }
+  return iter->second;*/
+}
+
+void RecordTypeInfo::serialize(::hadoop::OArchive& a_, const char* tag) const
+{
+  a_.startRecord(*this, tag);
+  // name
+  a_.serialize(name, tag);
+  /*// number of elements
+  a_.serialize((int32_t)typeInfos.size(), tag);
+  // write out each element
+  for (std::vector<FieldTypeInfo*>::const_iterator iter=typeInfos.begin();
+       iter!=typeInfos.end(); ++iter) {
+    (*iter)->serialize(a_, tag);
+    }*/
+  pStid->serializeRest(a_, tag);
+  a_.endRecord(*this, tag);
+}
+
+void RecordTypeInfo::print(int space) const
+{
+  for (int i=0; i<space; i++) {
+    printf(" ");
+  }
+  printf("RecordTypeInfo::%s\n", name.c_str());
+  pStid->print(space);
+  /*for (unsigned i=0; i<typeInfos.size(); i++) {
+    typeInfos[i]->print(space+2);
+    }*/
+}
+
+void RecordTypeInfo::deserialize(::hadoop::IArchive& a_, const char* tag)
+{
+  a_.startRecord(*this, tag);
+  // name
+  a_.deserialize(name, tag);
+  pStid->deserialize(a_, tag);
+  a_.endRecord(*this, tag);
+}
+

Added: lucene/hadoop/trunk/src/c++/librecordio/recordTypeInfo.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/recordTypeInfo.hh?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/recordTypeInfo.hh (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/recordTypeInfo.hh Thu Jan 17 01:43:33 2008
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RECORDTYPEINFO_HH_
+#define RECORDTYPEINFO_HH_
+
+#include "recordio.hh"
+#include <vector>
+#include <map>
+#include "fieldTypeInfo.hh"
+
+namespace hadoop {
+
+class RecordTypeInfo : public ::hadoop::Record {
+
+private: 
+  //std::vector<FieldTypeInfo* > typeInfos;
+  std::string name;
+  //std::map<std::string, RecordTypeInfo*> structRTIs;
+  StructTypeID *pStid;
+
+  RecordTypeInfo(const char * pName, StructTypeID* pStid): name(pName),pStid(pStid) {}
+
+public: 
+  RecordTypeInfo();
+  RecordTypeInfo(const char *pName);
+  //RecordTypeInfo(const RecordTypeInfo& rti);
+  virtual ~RecordTypeInfo();
+
+  void addField(const std::string* pFieldID, const TypeID* pTypeID);
+  void addAll(std::vector<FieldTypeInfo*>& vec);
+  const std::vector<FieldTypeInfo*>& getFieldTypeInfos() const;
+  void serialize(::hadoop::OArchive& a_, const char* tag) const;
+  void deserialize(::hadoop::IArchive& a_, const char* tag);
+  RecordTypeInfo* clone() const {return new RecordTypeInfo(*this);}
+  RecordTypeInfo* getNestedStructTypeInfo(const char *structName) const;
+
+  const ::std::string& getName() const {return name;}
+  void setName(const ::std::string& name) {this->name = name;}
+
+  const ::std::string& type() const {return name;}
+  const ::std::string& signature() const {return name;}
+
+  void print(int space=0) const;
+
+
+};
+
+
+
+}
+#endif // RECORDTYPEINFO_HH_
+

Modified: lucene/hadoop/trunk/src/c++/librecordio/recordio.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/recordio.cc?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/recordio.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/recordio.cc Thu Jan 17 01:43:33 2008
@@ -68,7 +68,7 @@
   delete mpArchive;
 }
 
-void hadoop::RecordWriter::write(Record& record)
+void hadoop::RecordWriter::write(const Record& record)
 {
   record.serialize(*mpArchive, (const char*) NULL);
 }

Modified: lucene/hadoop/trunk/src/c++/librecordio/recordio.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/recordio.hh?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/recordio.hh (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/recordio.hh Thu Jan 17 01:43:33 2008
@@ -70,7 +70,7 @@
   OArchive* mpArchive;
 public:
   RecordWriter(OutStream& stream, RecFormat f);
-  virtual void write(hadoop::Record& record);
+  virtual void write(const hadoop::Record& record);
   virtual ~RecordWriter();
 };
 }; // end namspace hadoop

Modified: lucene/hadoop/trunk/src/c++/librecordio/test/test.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/test/test.cc?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/test/test.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/test/test.cc Thu Jan 17 01:43:33 2008
@@ -17,11 +17,12 @@
  */
 
 #include "test.hh"
+#include <vector>
 
 int main()
 {
-  org::apache::hadoop::record::test::TestRecord1 r1;
-  org::apache::hadoop::record::test::TestRecord1 r2;
+  org::apache::hadoop::record::test::RecRecord1 r1;
+  org::apache::hadoop::record::test::RecRecord1 r2;
   {
     hadoop::FileOutStream ostream;
     ostream.open("/tmp/hadooptmp.dat", true);
@@ -97,6 +98,212 @@
     }
     istream.close();
   }
+  
+  /* 
+   * Tests to check for versioning functionality
+   */
+  
+  // basic test
+  // write out a record and its type info, read it back using its typeinfo
+  {
+    hadoop::FileOutStream ostream, ortistream;
+    ostream.open("/tmp/hadooptmp.dat", true);
+    ortistream.open("/tmp/hadooprti.dat", true);
+    hadoop::RecordWriter writer(ostream, hadoop::kBinary);
+    hadoop::RecordWriter writerRti(ortistream, hadoop::kBinary);
+    r1.setBoolVal(true);
+    r1.setByteVal((int8_t)0x66);
+    r1.setFloatVal(3.145);
+    r1.setDoubleVal(1.5234);
+    r1.setIntVal(4567);
+    r1.setLongVal(0x5a5a5a5a5a5aLL);
+    std::string& s = r1.getStringVal();
+    s = "random text";
+    writer.write(r1);
+    ostream.close();
+    // write out rti info
+    writerRti.write(org::apache::hadoop::record::test::RecRecord1::getTypeInfo());
+    ortistream.close();
+
+    // read
+    hadoop::FileInStream istream;
+    istream.open("/tmp/hadooptmp.dat");
+    hadoop::RecordReader reader(istream, hadoop::kBinary);
+    hadoop::FileInStream irtistream;
+    irtistream.open("/tmp/hadooprti.dat");
+    hadoop::RecordReader readerRti(irtistream, hadoop::kBinary);
+    hadoop::RecordTypeInfo rti;
+    readerRti.read(rti);
+    irtistream.close();
+    org::apache::hadoop::record::test::RecRecord1::setTypeFilter(rti);
+    reader.read(r2);
+    if (r1 == r2) {
+      printf("Basic versioning test passed.\n");
+    } else {
+      printf("Basic versioning test failed.\n");
+    }
+    istream.close();
+  }     
+  
+  // versioning:write out a record and its type info, read back a similar record using the written record's typeinfo
+  {
+    hadoop::FileOutStream ostream, ortistream;
+    ostream.open("/tmp/hadooptmp.dat", true);
+    ortistream.open("/tmp/hadooprti.dat", true);
+    hadoop::RecordWriter writer(ostream, hadoop::kBinary);
+    hadoop::RecordWriter writerRti(ortistream, hadoop::kBinary);
+
+    // we create an array of records to write
+    std::vector<org::apache::hadoop::record::test::RecRecordOld*> recsWrite;
+    int i, j, k, l;
+    char buf[1000];
+    for (i=0; i<5; i++) {
+      org::apache::hadoop::record::test::RecRecordOld* ps1Rec = 
+        new org::apache::hadoop::record::test::RecRecordOld();
+      sprintf(buf, "This is record s1: %d", i);
+      ps1Rec->getName().assign(buf);
+
+      for (j=0; j<3; j++) {
+        ps1Rec->getIvec().push_back((int64_t)(i+j));
+      }
+
+      for (j=0; j<2; j++) {
+        std::vector<org::apache::hadoop::record::test::RecRecord0>* pVec = 
+          new std::vector<org::apache::hadoop::record::test::RecRecord0>();
+        for (k=0; k<3; k++) {
+          org::apache::hadoop::record::test::RecRecord0 *psRec = 
+            new org::apache::hadoop::record::test::RecRecord0();
+          sprintf(buf, "This is record s: (%d: %d)", j, k);
+          psRec->getStringVal().assign(buf);
+        }
+        ps1Rec->getSvec().push_back(*pVec);
+      }
+
+      sprintf(buf, "This is record s: %d", i);
+      ps1Rec->getInner().getStringVal().assign(buf);
+
+      for (l=0; l<2; l++) {
+        std::vector<std::vector<std::string> >* ppVec =
+          new std::vector<std::vector<std::string> >();
+        for (j=0; j<2; j++) {
+          std::vector< std::string >* pVec =
+            new std::vector< std::string >();
+          for (k=0; k<3; k++) {
+            sprintf(buf, "THis is a nested string: (%d: %d: %d)", l, j, k);
+            std::string* s = new std::string((const char*)buf);
+            pVec->push_back(*s);
+          }
+        }
+        ps1Rec->getStrvec().push_back(*ppVec);
+      }
+
+      ps1Rec->setI1(100+i);
+
+      ps1Rec->getMap1()[23] = "23";
+      ps1Rec->getMap1()[11] = "11";
+
+      std::map<int32_t, int64_t>* m1 = new std::map<int32_t, int64_t>();
+      std::map<int32_t, int64_t>* m2 = new std::map<int32_t, int64_t>();
+      (*m1)[5] = 5;
+      (*m1)[10] = 10;
+      (*m2)[15] = 15;
+      (*m2)[20] = 20;
+      ps1Rec->getMvec1().push_back(*m1);
+      ps1Rec->getMvec1().push_back(*m2);
+      ps1Rec->getMvec2().push_back(*m1);
+
+      recsWrite.push_back(ps1Rec);
+    }
+
+    // write out to file
+    for (unsigned int i=0; i<recsWrite.size(); i++) {
+      writer.write(*(recsWrite[i]));
+    }
+    ostream.close();
+    // write out rti info
+    writerRti.write(org::apache::hadoop::record::test::RecRecordOld::getTypeInfo());
+    ortistream.close();
+
+    // read
+    hadoop::FileInStream istream;
+    istream.open("/tmp/hadooptmp.dat");
+    hadoop::RecordReader reader(istream, hadoop::kBinary);
+    hadoop::FileInStream irtistream;
+    irtistream.open("/tmp/hadooprti.dat");
+    hadoop::RecordReader readerRti(irtistream, hadoop::kBinary);
+    hadoop::RecordTypeInfo rti;
+    readerRti.read(rti);
+    irtistream.close();
+    org::apache::hadoop::record::test::RecRecordNew::setTypeFilter(rti);
+    
+    // read records
+    std::vector<org::apache::hadoop::record::test::RecRecordNew*> recsRead;
+    for (unsigned int i=0; i<recsWrite.size(); i++) {
+      org::apache::hadoop::record::test::RecRecordNew* ps2Rec = 
+        new org::apache::hadoop::record::test::RecRecordNew();
+      reader.read(*ps2Rec);
+      recsRead.push_back(ps2Rec);
+    }
+    istream.close();
+
+    // compare
+    bool pass = true;
+    for (unsigned int i=0; i<recsRead.size(); i++) {
+      org::apache::hadoop::record::test::RecRecordNew* ps2In = recsRead[i];
+      org::apache::hadoop::record::test::RecRecordOld* ps1Out = recsWrite[i];
+
+      if (!ps2In->getName2().empty()) {
+        printf("Error in s2: name2\n");
+        pass = false;
+      }
+
+      if (!(ps2In->getInner() == ps1Out->getInner())) {
+        printf("error in s2: s1 struct\n");
+        pass = false;
+      }
+
+      if (0 != ps2In->getIvec().size()) {
+        printf("error in s2: ivec\n");
+        pass = false;
+      }
+
+      if (0 != ps2In->getSvec().size()) {
+        printf("error in s2: svec\n");
+        pass = false;
+      }
+
+      for (unsigned int j=0; j<ps2In->getStrvec().size(); j++) {
+        ::std::vector< ::std::vector< ::std::string > >& ss2Vec = ps2In->getStrvec()[j];
+        ::std::vector< ::std::vector< ::std::string > >& ss1Vec = ps1Out->getStrvec()[j];
+        for (unsigned int k=0; k<ss2Vec.size(); k++) {
+          ::std::vector< ::std::string >& s2Vec = ss2Vec[k];
+          ::std::vector< ::std::string >& s1Vec = ss1Vec[k];
+          for (unsigned int l=0; l<s2Vec.size(); l++) {
+            if (s2Vec[l] != s1Vec[l]) {
+              printf("Error in s2: s2Vec\n");
+              pass = false;
+            }
+          }
+        }
+      }
+
+      if (0 != ps2In->getMap1().size()) {
+        printf("Error in s2: map1\n");
+        pass = false;
+      }
+
+      for (unsigned int j=0; j<ps2In->getMvec2().size(); j++) {
+        if (ps2In->getMvec2()[j] != ps1Out->getMvec2()[j]) {
+          printf("Error in s2: mvec2\n");
+          pass = false;
+        }
+      }
+    }
+  
+    if (pass)   
+      printf("Versioning test passed.\n");
+  }     
+   
   return 0;
 }
 

Modified: lucene/hadoop/trunk/src/c++/librecordio/test/test.jr
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/test/test.jr?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/test/test.jr (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/test/test.jr Thu Jan 17 01:43:33 2008
@@ -17,11 +17,11 @@
  */
 
 module org.apache.hadoop.record.test {
-    class TestRecord0 {
+    class RecRecord0 {
         ustring     StringVal;
     }
 
-    class TestRecord1 {
+    class RecRecord1 {
         boolean         BoolVal;
         byte            ByteVal;
         int             IntVal;
@@ -33,5 +33,31 @@
         vector<ustring> VectorVal;
         map<ustring, ustring>   MapVal;
     }
+    
+  class RecRecordOld {
+  	ustring name;
+  	vector<long> ivec;
+  	vector<vector<RecRecord0>> svec;
+  	RecRecord0 inner;
+  	vector<vector<vector<ustring>>> strvec;
+  	float i1;
+  	map<byte, ustring> map1;
+  	vector<map<int, long>> mvec1;
+  	vector<map<int, long>> mvec2;
+  }
+  
+  /* RecRecordNew is a lot like RecRecordOld. Helps test for versioning. */
+  class RecRecordNew {
+  	ustring name2;
+  	RecRecord0 inner;
+  	vector<int> ivec;
+  	vector<vector<int>> svec;
+  	vector<vector<vector<ustring>>> strvec;
+		int i1;  	
+		map<long, ustring> map1;
+  	vector<map<int, long>> mvec2;	
+  }
+  
+    
 }
 

Modified: lucene/hadoop/trunk/src/c++/librecordio/test/testFromJava.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/test/testFromJava.cc?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/test/testFromJava.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/test/testFromJava.cc Thu Jan 17 01:43:33 2008
@@ -20,8 +20,8 @@
 
 int main()
 {
-  org::apache::hadoop::record::test::TestRecord1 r1;
-  org::apache::hadoop::record::test::TestRecord1 r2;
+  org::apache::hadoop::record::test::RecRecord1 r1;
+  org::apache::hadoop::record::test::RecRecord1 r2;
   r1.setBoolVal(true);
   r1.setByteVal((int8_t)0x66);
   r1.setFloatVal(3.145);

Added: lucene/hadoop/trunk/src/c++/librecordio/typeIDs.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/typeIDs.cc?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/typeIDs.cc (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/typeIDs.cc Thu Jan 17 01:43:33 2008
@@ -0,0 +1,274 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "typeIDs.hh"
+
+using namespace hadoop;
+
+void TypeID::serialize(::hadoop::OArchive& a_, const char* tag) const
+{
+  a_.serialize(typeVal, tag);
+}
+
+bool TypeID::operator==(const TypeID& peer_) const 
+{
+  return (this->typeVal == peer_.typeVal);
+}
+
+void TypeID::print(int space) const
+{
+  for (int i=0; i<space; i++) {
+    printf(" ");
+  }
+  printf("typeID(%lx) = %d\n", (long)this, typeVal);
+}
+
+
+/*StructTypeID::StructTypeID(const char *p): TypeID(RIOTYPE_STRUCT)
+{
+  pName = new std::string(p);
+}
+
+StructTypeID::StructTypeID(std::string* p): TypeID(RIOTYPE_STRUCT)
+{
+  this->pName = p;
+}*/
+
+StructTypeID::StructTypeID(const std::vector<FieldTypeInfo*>& vec) : 
+  TypeID(RIOTYPE_STRUCT) 
+{
+  // we need to copy object clones into our own vector
+  for (unsigned int i=0; i<vec.size(); i++) {
+    typeInfos.push_back(vec[i]->clone());
+  }
+}
+
+/*StructTypeID::StructTypeID(const StructTypeID& ti) :
+  TypeID(RIOTYPE_STRUCT)
+{
+  // we need to copy object clones into our own vector
+  for (unsigned int i=0; i<ti.typeInfos.size(); i++) {
+    typeInfos.push_back(ti.typeInfos[i]->clone());
+  }
+} */ 
+
+StructTypeID::~StructTypeID()
+{
+  for (unsigned int i=0; i<typeInfos.size(); i++) {
+    delete typeInfos[i];
+  }
+}  
+
+void StructTypeID::add(FieldTypeInfo *pti)
+{
+  typeInfos.push_back(pti);
+}
+
+// return the StructTypeiD, if any, of the given field
+StructTypeID* StructTypeID::findStruct(const char *pStructName)
+{
+  // walk through the list, searching. Not the most efficient way, but this
+  // in intended to be used rarely, so we keep it simple. 
+  // As an optimization, we can keep a hashmap of record name to its RTI, for later.
+  for (unsigned int i=0; i<typeInfos.size(); i++) {
+    if ((0 == typeInfos[i]->getFieldID()->compare(pStructName)) && 
+	(typeInfos[i]->getTypeID()->getTypeVal()==RIOTYPE_STRUCT)) {
+      return (StructTypeID*)(typeInfos[i]->getTypeID()->clone());
+    }
+  }
+  return NULL;
+}
+
+void StructTypeID::serialize(::hadoop::OArchive& a_, const char* tag) const
+{
+  a_.serialize(typeVal, tag);
+  serializeRest(a_, tag);
+}
+
+/* 
+ * Writes rest of the struct (excluding type value).
+ * As an optimization, this method is directly called by RTI 
+ * for the top level record so that we don't write out the byte
+ * indicating that this is a struct (since top level records are
+ * always structs).
+ */
+void StructTypeID::serializeRest(::hadoop::OArchive& a_, const char* tag) const
+{
+  a_.serialize((int32_t)typeInfos.size(), tag);
+  for (unsigned int i=0; i<typeInfos.size(); i++) {
+    typeInfos[i]->serialize(a_, tag);
+  }
+}
+
+
+/* 
+ * deserialize ourselves. Called by RTI. 
+ */
+void StructTypeID::deserialize(::hadoop::IArchive& a_, const char* tag)
+{
+  // number of elements
+  int numElems;
+  a_.deserialize(numElems, tag);
+  for (int i=0; i<numElems; i++) {
+    typeInfos.push_back(genericReadTypeInfo(a_, tag));
+  }
+}
+
+// generic reader: reads the next TypeInfo object from stream and returns it
+FieldTypeInfo* StructTypeID::genericReadTypeInfo(::hadoop::IArchive& a_, const char* tag)
+{
+  // read name of field
+  std::string*  pName = new std::string();
+  a_.deserialize(*pName, tag);
+  TypeID* pti = genericReadTypeID(a_, tag);
+  return new FieldTypeInfo(pName, pti);
+}
+
+// generic reader: reads the next TypeID object from stream and returns it
+TypeID* StructTypeID::genericReadTypeID(::hadoop::IArchive& a_, const char* tag)
+{
+  int8_t typeVal;
+  a_.deserialize(typeVal, tag);
+  switch(typeVal) {
+  case RIOTYPE_BOOL: 
+  case RIOTYPE_BUFFER: 
+  case RIOTYPE_BYTE: 
+  case RIOTYPE_DOUBLE: 
+  case RIOTYPE_FLOAT: 
+  case RIOTYPE_INT: 
+  case RIOTYPE_LONG: 
+  case RIOTYPE_STRING: 
+    return new TypeID(typeVal);
+  case RIOTYPE_STRUCT: 
+    {
+      StructTypeID* pstID = new StructTypeID();
+      int numElems;
+      a_.deserialize(numElems, tag);
+      for (int i=0; i<numElems; i++) {
+	pstID->add(genericReadTypeInfo(a_, tag));
+      }
+      return pstID;
+    }
+  case RIOTYPE_VECTOR: 
+    {
+      TypeID* pti = genericReadTypeID(a_, tag);
+      return new VectorTypeID(pti);
+    }
+  case RIOTYPE_MAP: 
+    {
+      TypeID* ptiKey = genericReadTypeID(a_, tag);
+      TypeID* ptiValue = genericReadTypeID(a_, tag);
+      return new MapTypeID(ptiKey, ptiValue);
+    }
+  default: 
+    // shouldn't be here
+    return NULL;
+  }
+}
+
+void StructTypeID::print(int space) const
+{
+  TypeID::print(space);
+  for (int i=0; i<space; i++) {
+    printf(" ");
+  }
+  printf("StructTypeInfo(%lx): \n", (long)&typeInfos);
+  for (unsigned int i=0; i<typeInfos.size(); i++) {
+    typeInfos[i]->print(space+2);
+  }
+}
+
+
+VectorTypeID::~VectorTypeID()
+{
+  delete ptiElement;
+}
+
+VectorTypeID::VectorTypeID(const VectorTypeID& ti): TypeID(RIOTYPE_VECTOR)
+{
+  ptiElement = ti.ptiElement->clone();
+}
+
+void VectorTypeID::serialize(::hadoop::OArchive& a_, const char* tag) const
+{
+  a_.serialize(typeVal, tag);
+  ptiElement->serialize(a_, tag);
+}
+
+bool VectorTypeID::operator==(const TypeID& peer_) const
+{
+  if (typeVal != peer_.getTypeVal()) {
+    return false;
+  }
+  // this must be a vector type id
+  return (*ptiElement) == (*((VectorTypeID&)peer_).ptiElement);
+}
+
+void VectorTypeID::print(int space) const
+{
+  TypeID::print(space);
+  for (int i=0; i<space; i++) {
+    printf(" ");
+  }
+  printf("VectorTypeInfo(%lx): \n", (long)this);
+  ptiElement->print(space+2);
+}
+
+
+MapTypeID::~MapTypeID()
+{
+  delete ptiKey;
+  delete ptiValue;
+}
+
+MapTypeID::MapTypeID(const MapTypeID& ti): TypeID(RIOTYPE_MAP)
+{
+  ptiKey = ti.ptiKey->clone();
+  ptiValue = ti.ptiValue->clone();
+}
+
+void MapTypeID::serialize(::hadoop::OArchive& a_, const char* tag) const
+{
+  a_.serialize(typeVal, tag);
+  ptiKey->serialize(a_, tag);
+  ptiValue->serialize(a_, tag);
+}
+
+bool MapTypeID::operator==(const TypeID& peer_) const
+{
+  if (typeVal != peer_.getTypeVal()) {
+    return false;
+  }
+  // this must be a map type id
+  MapTypeID& mti = (MapTypeID&) peer_;
+  if (!(*ptiKey == *(mti.ptiKey))) {
+    return false;
+  }
+  return ((*ptiValue == *(mti.ptiValue)));
+}
+
+void MapTypeID::print(int space) const
+{
+  TypeID::print(space);
+  for (int i=0; i<space; i++) {
+    printf(" ");
+  }
+  printf("MapTypeInfo(%lx): \n", (long)this);
+  ptiKey->print(space+2);
+  ptiValue->print(space+2);
+}

Added: lucene/hadoop/trunk/src/c++/librecordio/typeIDs.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/typeIDs.hh?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/typeIDs.hh (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/typeIDs.hh Thu Jan 17 01:43:33 2008
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef TYPEIDS_HH_
+#define TYPEIDS_HH_
+
+#include "recordio.hh"
+#include "fieldTypeInfo.hh"
+
+namespace hadoop {
+
+class FieldTypeInfo;
+
+/* 
+ * enum of types. We define assign values to individual bytes, rather
+ * than use enums because we want to make teh values consistent with
+ * Java code, so we need to control the values.
+ */
+const int8_t RIOTYPE_BOOL   = 1;
+const int8_t RIOTYPE_BUFFER = 2;
+const int8_t RIOTYPE_BYTE   = 3;
+const int8_t RIOTYPE_DOUBLE = 4;
+const int8_t RIOTYPE_FLOAT  = 5;
+const int8_t RIOTYPE_INT    = 6;
+const int8_t RIOTYPE_LONG   = 7;
+const int8_t RIOTYPE_MAP    = 8;
+const int8_t RIOTYPE_STRING = 9;
+const int8_t RIOTYPE_STRUCT = 10;
+const int8_t RIOTYPE_VECTOR = 11;
+
+
+
+/* 
+ * Represents typeID for basic types. 
+ * Serializes just the single int8_t.
+ */
+class TypeID {
+
+public: 
+
+  TypeID(int8_t typeVal) {this->typeVal = typeVal;}
+  TypeID(const TypeID& t) {this->typeVal = t.typeVal;}
+  virtual ~TypeID() {}
+
+  int8_t getTypeVal() const {return typeVal;}
+  virtual void serialize(::hadoop::OArchive& a_, const char* tag) const;
+
+  virtual bool operator==(const TypeID& peer_) const;
+  virtual TypeID* clone() const {return new TypeID(*this);}
+
+  virtual void print(int space=0) const;
+  
+protected: 
+  int8_t typeVal;
+};
+
+
+/* 
+ * no predefined TypeID objects, since memory management becomes difficult. 
+ * If some TypeID objects are consts and others are new-ed, becomes hard to 
+ * destroy const objects without reference counting. 
+ */
+/*const TypeID TID_BoolTypeID(RIOTYPE_BOOL);
+const TypeID TID_BufferTypeID(RIOTYPE_BUFFER);
+const TypeID TID_ByteTypeID(RIOTYPE_BYTE);
+const TypeID TID_DoubleTypeID(RIOTYPE_DOUBLE);
+const TypeID TID_FloatTypeID(RIOTYPE_FLOAT);
+const TypeID TID_IntTypeID(RIOTYPE_INT);
+const TypeID TID_LongTypeID(RIOTYPE_LONG);
+const TypeID TID_StringTypeID(RIOTYPE_STRING);*/
+
+
+/* 
+ * TypeID for structures
+ */
+class StructTypeID : public TypeID {
+
+private: 
+  // note: we own the memory mgmt of TypeInfo objects stored in the vector
+  std::vector<FieldTypeInfo*> typeInfos;
+  FieldTypeInfo* genericReadTypeInfo(::hadoop::IArchive& a_, const char* tag);
+  TypeID* genericReadTypeID(::hadoop::IArchive& a_, const char* tag);
+
+public: 
+  /*StructTypeID(const char* p);
+  StructTypeID(std::string* p);
+  StructTypeID(const StructTypeID& ti);*/
+  StructTypeID(): TypeID(RIOTYPE_STRUCT) {};
+  StructTypeID(const std::vector<FieldTypeInfo*>& vec);
+  virtual ~StructTypeID();
+
+  void add(FieldTypeInfo *pti);
+  std::vector<FieldTypeInfo*>& getFieldTypeInfos() {return typeInfos;}
+  StructTypeID* findStruct(const char *pStructName);
+  void serialize(::hadoop::OArchive& a_, const char* tag) const;
+  void serializeRest(::hadoop::OArchive& a_, const char* tag) const;
+  void deserialize(::hadoop::IArchive& a_, const char* tag);
+  virtual TypeID* clone() const {return new StructTypeID(*this);}
+
+  virtual void print(int space=0) const;
+
+};
+
+
+/* 
+ * TypeID for vectors
+ */
+class VectorTypeID : public TypeID {
+
+private: 
+  // ptiElement's memory mgmt is owned by class
+  TypeID* ptiElement;
+
+public: 
+  VectorTypeID(TypeID* ptiElement): TypeID(RIOTYPE_VECTOR), ptiElement(ptiElement) {}
+  VectorTypeID(const VectorTypeID& ti);
+  virtual ~VectorTypeID();
+
+  const TypeID* getElementTypeID() {return ptiElement;}
+  virtual TypeID* clone() const {return new VectorTypeID(*this);}
+  void serialize(::hadoop::OArchive& a_, const char* tag) const;
+  virtual bool operator==(const TypeID& peer_) const;
+  
+  virtual void print(int space=0) const;
+};
+
+/* 
+ * TypeID for maps
+ */
+class MapTypeID : public TypeID {
+
+private: 
+  // ptiKay and ptiValue's memory mgmt is owned by class
+  TypeID* ptiKey;
+  TypeID* ptiValue;
+
+public: 
+  MapTypeID(TypeID* ptiKey, TypeID* ptiValue): 
+    TypeID(RIOTYPE_MAP), ptiKey(ptiKey), ptiValue(ptiValue) {}
+  MapTypeID(const MapTypeID& ti);
+  virtual ~MapTypeID();
+
+  const TypeID* getKeyTypeID() {return ptiKey;}
+  const TypeID* getValueTypeID() {return ptiValue;}
+  virtual TypeID* clone() const {return new MapTypeID(*this);}
+  void serialize(::hadoop::OArchive& a_, const char* tag) const;
+  virtual bool operator==(const TypeID& peer_) const;
+  
+  virtual void print(int space=0) const;
+};
+
+}
+#endif // TYPEIDS_HH_
+

Added: lucene/hadoop/trunk/src/c++/librecordio/typeInfo.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/typeInfo.cc?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/typeInfo.cc (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/typeInfo.cc Thu Jan 17 01:43:33 2008
@@ -0,0 +1,52 @@
+
+#include "typeInfo.hh"
+
+using namespace hadoop;
+
+TypeInfo::~TypeInfo()
+{
+  delete pFieldID;
+  delete pTypeID;
+}
+
+/*TypeInfo& TypeInfo::operator =(const TypeInfo& ti) {
+  pFieldID = ti.pFieldID;
+  pTypeID = ti.pTypeID;
+  return *this;
+  }*/
+
+TypeInfo::TypeInfo(const TypeInfo& ti)
+{
+  pFieldID = new std::string(*ti.pFieldID);
+  pTypeID = ti.pTypeID->clone();
+}
+
+
+void TypeInfo::serialize(::hadoop::OArchive& a_, const char* tag) const
+{
+  a_.serialize(*pFieldID, tag);
+  pTypeID->serialize(a_, tag);
+}
+
+bool TypeInfo::operator==(const TypeInfo& peer_) const 
+{
+  // first check if fieldID matches
+  if (0 != pFieldID->compare(*(peer_.pFieldID))) {
+    return false;
+  }
+  // now see if typeID matches
+  return (*pTypeID == *(peer_.pTypeID));
+}
+
+void TypeInfo::print(int space) const
+{
+  for (int i=0; i<space; i++) {
+    printf(" ");
+  }
+  printf("TypeInfo(%lx):\n", (long)this);
+  for (int i=0; i<space+2; i++) {
+    printf(" ");
+  }
+  printf("field = \"%s\"\n", pFieldID->c_str());
+  pTypeID->print(space+2);
+}

Added: lucene/hadoop/trunk/src/c++/librecordio/typeInfo.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/typeInfo.hh?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/typeInfo.hh (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/typeInfo.hh Thu Jan 17 01:43:33 2008
@@ -0,0 +1,38 @@
+#ifndef TYPEINFO_HH_
+#define TYPEINFO_HH_
+
+#include "recordio.hh"
+#include "typeIDs.hh"
+
+namespace hadoop {
+
+class TypeID;
+
+class TypeInfo {
+  
+private: 
+  // we own memory mgmt of these vars
+  const std::string* pFieldID;
+  const TypeID* pTypeID;
+
+public: 
+  TypeInfo(const std::string* pFieldID, const TypeID* pTypeID) : 
+    pFieldID(pFieldID), pTypeID(pTypeID) {}
+  TypeInfo(const TypeInfo& ti);
+  virtual ~TypeInfo();
+
+  const TypeID* getTypeID() const {return pTypeID;}
+  const std::string* getFieldID() const {return pFieldID;}
+  void serialize(::hadoop::OArchive& a_, const char* tag) const;
+  bool operator==(const TypeInfo& peer_) const;
+  TypeInfo* clone() const {return new TypeInfo(*this);}
+
+  //TypeInfo& operator =(const TypeInfo& ti);
+  void print(int space=0) const;
+
+};
+
+}
+
+#endif // TYPEINFO_HH_
+

Added: lucene/hadoop/trunk/src/c++/librecordio/utils.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/utils.cc?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/utils.cc (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/utils.cc Thu Jan 17 01:43:33 2008
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "utils.hh"
+#include "recordTypeInfo.hh"
+
+using namespace hadoop;
+
+void Utils::skip(IArchive& a, const char* tag, const TypeID& typeID)
+{
+  bool b;
+  size_t len=0;
+  ::std::string str;
+  int8_t bt;
+  double d;
+  float f;
+  int32_t i;
+  int64_t l;
+
+  switch(typeID.getTypeVal()) {
+  case RIOTYPE_BOOL: 
+    a.deserialize(b, tag);
+    break;
+  case RIOTYPE_BUFFER: 
+    a.deserialize(str, len, tag);
+    break;
+  case RIOTYPE_BYTE: 
+    a.deserialize(bt, tag);
+    break;
+  case RIOTYPE_DOUBLE: 
+    a.deserialize(d, tag);
+    break;
+  case RIOTYPE_FLOAT: 
+    a.deserialize(f, tag);
+    break;
+  case RIOTYPE_INT: 
+    a.deserialize(i, tag);
+    break;
+  case RIOTYPE_LONG: 
+    a.deserialize(l, tag);
+    break;
+  case RIOTYPE_MAP:
+    {
+      // since we don't know the key, value types, 
+      // we need to deserialize in a generic manner
+      Index* idx = a.startMap(tag);
+      MapTypeID& mtID = (MapTypeID&) typeID;
+      while (!idx->done()) {
+	skip(a, tag, *(mtID.getKeyTypeID()));
+	skip(a, tag, *(mtID.getValueTypeID()));
+	idx->incr();
+      }
+      a.endMap(idx, tag);
+    }
+    break;
+  case RIOTYPE_STRING: 
+    a.deserialize(str, tag);
+    break;
+  case RIOTYPE_STRUCT: 
+    {
+      // since we don't know the key, value types, 
+      // we need to deserialize in a generic manner
+      // we need to pass a record in, though it's never used
+      RecordTypeInfo rec;
+      a.startRecord(rec, tag);
+      StructTypeID& stID = (StructTypeID&) typeID;
+      std::vector<FieldTypeInfo*>& typeInfos = stID.getFieldTypeInfos();
+      for (unsigned int i=0; i<typeInfos.size(); i++) {
+	skip(a, tag, *(typeInfos[i]->getTypeID()));
+      }
+      a.endRecord(rec, tag);
+    }
+    break;
+  case RIOTYPE_VECTOR:
+    {
+      // since we don't know the key, value types, 
+      // we need to deserialize in a generic manner
+      Index* idx = a.startVector(tag);
+      VectorTypeID& vtID = (VectorTypeID&) typeID;
+      while (!idx->done()) {
+	skip(a, tag, *(vtID.getElementTypeID()));
+	idx->incr();
+      }
+      a.endVector(idx, tag);
+    }
+    break;
+  default: 
+    // shouldn't be here
+    throw new IOException("Unknown typeID when skipping bytes");
+    break;
+  };
+
+}
+

Added: lucene/hadoop/trunk/src/c++/librecordio/utils.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/utils.hh?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/utils.hh (added)
+++ lucene/hadoop/trunk/src/c++/librecordio/utils.hh Thu Jan 17 01:43:33 2008
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef UTILS_HH_
+#define UTILS_HH_
+
+#include "recordio.hh"
+#include "typeIDs.hh"
+
+
+namespace hadoop {
+
+  /**
+   * Various utility functions for Hadooop record I/O platform.
+   */
+
+class Utils {
+
+private: 
+  /** Cannot create a new instance of Utils */
+  Utils() {};
+
+public: 
+
+  /**
+   * read/skip bytes from stream based on a type
+   */
+  static void skip(IArchive& a, const char* tag, const TypeID& typeID);
+
+};
+
+
+}
+#endif // UTILS_HH_
+

Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/Consts.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/Consts.java?rev=612774&view=auto
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/Consts.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/Consts.java Thu Jan 17 01:43:33 2008
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.record.compiler;
+
+import java.io.IOException;
+import java.util.Iterator;
+import org.apache.hadoop.record.RecordInput;
+
+/**
+ * const definitions for Record I/O compiler
+ */
+public class Consts {
+  
+  /** Cannot create a new instance */
+  private Consts() {
+  }
+  
+  // prefix to use for variables in generated classes
+  public static final String RIO_PREFIX = "_rio_";
+  // other vars used in generated classes
+  public static final String RTI_VAR = RIO_PREFIX + "recTypeInfo";
+  public static final String RTI_FILTER = RIO_PREFIX + "rtiFilter";
+  public static final String RTI_FILTER_FIELDS = RIO_PREFIX + "rtiFilterFields";
+  public static final String RECORD_OUTPUT = RIO_PREFIX + "a";
+  public static final String RECORD_INPUT = RIO_PREFIX + "a";
+  public static final String TAG = RIO_PREFIX + "tag";
+  
+}

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java Thu Jan 17 01:43:33 2008
@@ -48,11 +48,13 @@
     hh.write("#ifndef __"+fileName.toUpperCase().replace('.','_')+"__\n");
     hh.write("#define __"+fileName.toUpperCase().replace('.','_')+"__\n");
     hh.write("#include \"recordio.hh\"\n");
+    hh.write("#include \"recordTypeInfo.hh\"\n");
     for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
       hh.write("#include \""+iter.next().getName()+".hh\"\n");
     }
     
     cc.write("#include \""+fileName+".hh\"\n");
+    cc.write("#include \"utils.hh\"\n");
     
     for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
       iter.next().genCppCode(hh, cc, options);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBoolean.java Thu Jan 17 01:43:33 2008
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.record.compiler;
 
-import org.apache.hadoop.record.compiler.JType.CType;
-import org.apache.hadoop.record.compiler.JType.CppType;
 
 /**
  */
@@ -28,15 +26,20 @@
   class JavaBoolean extends JType.JavaType {
     
     JavaBoolean() {
-      super("boolean", "Bool", "Boolean");
+      super("boolean", "Bool", "Boolean", "TypeID.RIOType.BOOL");
     }
     
     void genCompareTo(CodeBuffer cb, String fname, String other) {
-      cb.append("ret = ("+fname+" == "+other+")? 0 : ("+fname+"?1:-1);\n");
+      cb.append(Consts.RIO_PREFIX + "ret = ("+fname+" == "+other+")? 0 : ("+
+          fname+"?1:-1);\n");
     }
     
+    String getTypeIDObjectString() {
+      return "org.apache.hadoop.record.meta.TypeID.BoolTypeID";
+    }
+
     void genHashCode(CodeBuffer cb, String fname) {
-      cb.append("ret = ("+fname+")?0:1;\n");
+      cb.append(Consts.RIO_PREFIX + "ret = ("+fname+")?0:1;\n");
     }
     
     // In Binary format, boolean is written as byte. true = 1, false = 0
@@ -65,10 +68,21 @@
     }
   }
   
+  class CppBoolean extends CppType {
+    
+    CppBoolean() {
+      super("bool");
+    }
+    
+    String getTypeIDObjectString() {
+      return "new ::hadoop::TypeID(::hadoop::RIOTYPE_BOOL)";
+    }
+  }
+
   /** Creates a new instance of JBoolean */
   public JBoolean() {
     setJavaType(new JavaBoolean());
-    setCppType(new CppType("bool"));
+    setCppType(new CppBoolean());
     setCType(new CType());
   }
   

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JBuffer.java Thu Jan 17 01:43:33 2008
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.record.compiler;
 
-import org.apache.hadoop.record.compiler.JCompType.CCompType;
 
 /**
  * Code generator for "buffer" type.
@@ -28,19 +27,24 @@
   class JavaBuffer extends JavaCompType {
     
     JavaBuffer() {
-      super("org.apache.hadoop.record.Buffer", "Buffer", "org.apache.hadoop.record.Buffer");
+      super("org.apache.hadoop.record.Buffer", "Buffer", 
+          "org.apache.hadoop.record.Buffer", "TypeID.RIOType.BUFFER");
     }
     
+    String getTypeIDObjectString() {
+      return "org.apache.hadoop.record.meta.TypeID.BufferTypeID";
+    }
+
     void genCompareTo(CodeBuffer cb, String fname, String other) {
-      cb.append("ret = "+fname+".compareTo("+other+");\n");
+      cb.append(Consts.RIO_PREFIX + "ret = "+fname+".compareTo("+other+");\n");
     }
     
     void genEquals(CodeBuffer cb, String fname, String peer) {
-      cb.append("ret = "+fname+".equals("+peer+");\n");
+      cb.append(Consts.RIO_PREFIX + "ret = "+fname+".equals("+peer+");\n");
     }
     
     void genHashCode(CodeBuffer cb, String fname) {
-      cb.append("ret = "+fname+".hashCode();\n");
+      cb.append(Consts.RIO_PREFIX + "ret = "+fname+".hashCode();\n");
     }
     
     void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
@@ -80,6 +84,11 @@
       cb.append("return "+fname+";\n");
       cb.append("}\n");
     }
+    
+    String getTypeIDObjectString() {
+      return "new ::hadoop::TypeID(::hadoop::RIOTYPE_BUFFER)";
+    }
+
   }
   /** Creates a new instance of JBuffer */
   public JBuffer() {

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JByte.java Thu Jan 17 01:43:33 2008
@@ -18,9 +18,6 @@
 
 package org.apache.hadoop.record.compiler;
 
-import org.apache.hadoop.record.compiler.JType.CType;
-import org.apache.hadoop.record.compiler.JType.CppType;
-
 /**
  * Code generator for "byte" type.
  */
@@ -29,9 +26,13 @@
   class JavaByte extends JavaType {
     
     JavaByte() {
-      super("byte", "Byte", "Byte");
+      super("byte", "Byte", "Byte", "TypeID.RIOType.BYTE");
     }
     
+    String getTypeIDObjectString() {
+      return "org.apache.hadoop.record.meta.TypeID.ByteTypeID";
+    }
+
     void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
       cb.append("{\n");
       cb.append("if ("+l+"<1) {\n");
@@ -56,9 +57,20 @@
     }
   }
   
+  class CppByte extends CppType {
+    
+    CppByte() {
+      super("int8_t");
+    }
+    
+    String getTypeIDObjectString() {
+      return "new ::hadoop::TypeID(::hadoop::RIOTYPE_BYTE)";
+    }
+  }
+
   public JByte() {
     setJavaType(new JavaByte());
-    setCppType(new CppType("int8_t"));
+    setCppType(new CppByte());
     setCType(new CType());
   }
   

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JCompType.java Thu Jan 17 01:43:33 2008
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.record.compiler;
 
+
 /**
  * Abstract base class for all the "compound" types such as ustring,
  * buffer, vector, map, and record.
@@ -26,28 +27,30 @@
   
   abstract class JavaCompType extends JavaType {
     
-    JavaCompType(String type, String suffix, String wrapper) {
-      super(type, suffix, wrapper);
+    JavaCompType(String type, String suffix, String wrapper, 
+        String typeIDByteString) { 
+      super(type, suffix, wrapper, typeIDByteString);
     }
     
     void genCompareTo(CodeBuffer cb, String fname, String other) {
-      cb.append("ret = "+fname+".compareTo("+other+");\n");
+      cb.append(Consts.RIO_PREFIX + "ret = "+fname+".compareTo("+other+");\n");
     }
     
     void genEquals(CodeBuffer cb, String fname, String peer) {
-      cb.append("ret = "+fname+".equals("+peer+");\n");
+      cb.append(Consts.RIO_PREFIX + "ret = "+fname+".equals("+peer+");\n");
     }
     
     void genHashCode(CodeBuffer cb, String fname) {
-      cb.append("ret = "+fname+".hashCode();\n");
+      cb.append(Consts.RIO_PREFIX + "ret = "+fname+".hashCode();\n");
     }
     
     void genClone(CodeBuffer cb, String fname) {
-      cb.append("other."+fname+" = ("+getType()+") this."+fname+".clone();\n");
+      cb.append(Consts.RIO_PREFIX + "other."+fname+" = ("+getType()+") this."+
+          fname+".clone();\n");
     }
   }
   
-  class CppCompType extends CppType {
+  abstract class CppCompType extends CppType {
     
     CppCompType(String type) {
       super(type);

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JDouble.java Thu Jan 17 01:43:33 2008
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.record.compiler;
 
-import org.apache.hadoop.record.compiler.JType.CType;
-import org.apache.hadoop.record.compiler.JType.CppType;
 
 /**
  */
@@ -28,12 +26,16 @@
   class JavaDouble extends JavaType {
     
     JavaDouble() {
-      super("double", "Double", "Double");
+      super("double", "Double", "Double", "TypeID.RIOType.DOUBLE");
     }
     
+    String getTypeIDObjectString() {
+      return "org.apache.hadoop.record.meta.TypeID.DoubleTypeID";
+    }
+
     void genHashCode(CodeBuffer cb, String fname) {
       String tmp = "Double.doubleToLongBits("+fname+")";
-      cb.append("ret = (int)("+tmp+"^("+tmp+">>>32));\n");
+      cb.append(Consts.RIO_PREFIX + "ret = (int)("+tmp+"^("+tmp+">>>32));\n");
     }
     
     void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
@@ -61,10 +63,23 @@
       cb.append("}\n");
     }
   }
+
+  class CppDouble extends CppType {
+    
+    CppDouble() {
+      super("double");
+    }
+    
+    String getTypeIDObjectString() {
+      return "new ::hadoop::TypeID(::hadoop::RIOTYPE_DOUBLE)";
+    }
+  }
+
+  
   /** Creates a new instance of JDouble */
   public JDouble() {
     setJavaType(new JavaDouble());
-    setCppType(new CppType("double"));
+    setCppType(new CppDouble());
     setCType(new CType());
   }
   

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFloat.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFloat.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFloat.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JFloat.java Thu Jan 17 01:43:33 2008
@@ -18,9 +18,6 @@
 
 package org.apache.hadoop.record.compiler;
 
-import org.apache.hadoop.record.compiler.JType.CType;
-import org.apache.hadoop.record.compiler.JType.CppType;
-
 /**
  */
 public class JFloat extends JType {
@@ -28,11 +25,15 @@
   class JavaFloat extends JavaType {
     
     JavaFloat() {
-      super("float", "Float", "Float");
+      super("float", "Float", "Float", "TypeID.RIOType.FLOAT");
     }
     
+    String getTypeIDObjectString() {
+      return "org.apache.hadoop.record.meta.TypeID.FloatTypeID";
+    }
+
     void genHashCode(CodeBuffer cb, String fname) {
-      cb.append("ret = Float.floatToIntBits("+fname+");\n");
+      cb.append(Consts.RIO_PREFIX + "ret = Float.floatToIntBits("+fname+");\n");
     }
     
     void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
@@ -60,10 +61,22 @@
       cb.append("}\n");
     }
   }
+
+  class CppFloat extends CppType {
+    
+    CppFloat() {
+      super("float");
+    }
+    
+    String getTypeIDObjectString() {
+      return "new ::hadoop::TypeID(::hadoop::RIOTYPE_FLOAT)";
+    }
+  }
+
   /** Creates a new instance of JFloat */
   public JFloat() {
     setJavaType(new JavaFloat());
-    setCppType(new CppType("float"));
+    setCppType(new CppFloat());
     setCType(new CType());
   }
   

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JInt.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JInt.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JInt.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JInt.java Thu Jan 17 01:43:33 2008
@@ -18,8 +18,6 @@
 
 package org.apache.hadoop.record.compiler;
 
-import org.apache.hadoop.record.compiler.JType.CType;
-import org.apache.hadoop.record.compiler.JType.CppType;
 
 /**
  * Code generator for "int" type
@@ -29,9 +27,13 @@
   class JavaInt extends JavaType {
     
     JavaInt() {
-      super("int", "Int", "Integer");
+      super("int", "Int", "Integer", "TypeID.RIOType.INT");
     }
     
+    String getTypeIDObjectString() {
+      return "org.apache.hadoop.record.meta.TypeID.IntTypeID";
+    }
+
     void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
       cb.append("{\n");
       cb.append("int i = org.apache.hadoop.record.Utils.readVInt("+b+", "+s+");\n");
@@ -53,10 +55,22 @@
       cb.append("}\n");
     }
   }
+
+  class CppInt extends CppType {
+    
+    CppInt() {
+      super("int32_t");
+    }
+    
+    String getTypeIDObjectString() {
+      return "new ::hadoop::TypeID(::hadoop::RIOTYPE_INT)";
+    }
+  }
+
   /** Creates a new instance of JInt */
   public JInt() {
     setJavaType(new JavaInt());
-    setCppType(new CppType("int32_t"));
+    setCppType(new CppInt());
     setCType(new CType());
   }
   

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JLong.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JLong.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JLong.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JLong.java Thu Jan 17 01:43:33 2008
@@ -18,9 +18,6 @@
 
 package org.apache.hadoop.record.compiler;
 
-import org.apache.hadoop.record.compiler.JType.CType;
-import org.apache.hadoop.record.compiler.JType.CppType;
-
 /**
  * Code generator for "long" type
  */
@@ -29,11 +26,16 @@
   class JavaLong extends JavaType {
     
     JavaLong() {
-      super("long", "Long", "Long");
+      super("long", "Long", "Long", "TypeID.RIOType.LONG");
     }
     
+    String getTypeIDObjectString() {
+      return "org.apache.hadoop.record.meta.TypeID.LongTypeID";
+    }
+
     void genHashCode(CodeBuffer cb, String fname) {
-      cb.append("ret = (int) ("+fname+"^("+fname+">>>32));\n");
+      cb.append(Consts.RIO_PREFIX + "ret = (int) ("+fname+"^("+
+          fname+">>>32));\n");
     }
     
     void genSlurpBytes(CodeBuffer cb, String b, String s, String l) {
@@ -57,10 +59,22 @@
       cb.append("}\n");
     }
   }
+
+  class CppLong extends CppType {
+    
+    CppLong() {
+      super("int64_t");
+    }
+    
+    String getTypeIDObjectString() {
+      return "new ::hadoop::TypeID(::hadoop::RIOTYPE_LONG)";
+    }
+  }
+
   /** Creates a new instance of JLong */
   public JLong() {
     setJavaType(new JavaLong());
-    setCppType(new CppType("int64_t"));
+    setCppType(new CppLong());
     setCType(new CType());
   }
   

Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java?rev=612774&r1=612773&r2=612774&view=diff
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java Thu Jan 17 01:43:33 2008
@@ -18,6 +18,9 @@
 
 package org.apache.hadoop.record.compiler;
 
+import java.util.Map;
+
+
 /**
  */
 public class JMap extends JCompType {
@@ -43,31 +46,48 @@
     JavaMap(JType.JavaType key, JType.JavaType value) {
       super("java.util.TreeMap<"+key.getWrapperType()+","+value.getWrapperType()+">",
             "Map",
-            "java.util.TreeMap<"+key.getWrapperType()+","+value.getWrapperType()+">");
+            "java.util.TreeMap<"+key.getWrapperType()+","+value.getWrapperType()+">",
+            "TypeID.RIOType.MAP");
       this.key = key;
       this.value = value;
     }
     
+    String getTypeIDObjectString() {
+      return "new org.apache.hadoop.record.meta.MapTypeID(" + 
+        key.getTypeIDObjectString() + ", " + 
+        value.getTypeIDObjectString() + ")";
+    }
+
+    void genSetRTIFilter(CodeBuffer cb, Map<String, Integer> nestedStructMap) {
+      key.genSetRTIFilter(cb, nestedStructMap);
+      value.genSetRTIFilter(cb, nestedStructMap);
+    }
+
     void genCompareTo(CodeBuffer cb, String fname, String other) {
       String setType = "java.util.Set<"+key.getWrapperType()+"> ";
       String iterType = "java.util.Iterator<"+key.getWrapperType()+"> ";
       cb.append("{\n");
-      cb.append(setType+getId("set1")+" = "+fname+".keySet();\n");
-      cb.append(setType+getId("set2")+" = "+other+".keySet();\n");
-      cb.append(iterType+getId("miter1")+" = "+
-                getId("set1")+".iterator();\n");
-      cb.append(iterType+getId("miter2")+" = "+
-                getId("set2")+".iterator();\n");
-      cb.append("for(; "+getId("miter1")+".hasNext() && "+
-                getId("miter2")+".hasNext();) {\n");
-      cb.append(key.getType()+" "+getId("k1")+
-                " = "+getId("miter1")+".next();\n");
-      cb.append(key.getType()+" "+getId("k2")+
-                " = "+getId("miter2")+".next();\n");
-      key.genCompareTo(cb, getId("k1"), getId("k2"));
-      cb.append("if (ret != 0) { return ret; }\n");
+      cb.append(setType+getId(Consts.RIO_PREFIX + "set1")+" = "+
+          fname+".keySet();\n");
+      cb.append(setType+getId(Consts.RIO_PREFIX + "set2")+" = "+
+          other+".keySet();\n");
+      cb.append(iterType+getId(Consts.RIO_PREFIX + "miter1")+" = "+
+                getId(Consts.RIO_PREFIX + "set1")+".iterator();\n");
+      cb.append(iterType+getId(Consts.RIO_PREFIX + "miter2")+" = "+
+                getId(Consts.RIO_PREFIX + "set2")+".iterator();\n");
+      cb.append("for(; "+getId(Consts.RIO_PREFIX + "miter1")+".hasNext() && "+
+                getId(Consts.RIO_PREFIX + "miter2")+".hasNext();) {\n");
+      cb.append(key.getType()+" "+getId(Consts.RIO_PREFIX + "k1")+
+                " = "+getId(Consts.RIO_PREFIX + "miter1")+".next();\n");
+      cb.append(key.getType()+" "+getId(Consts.RIO_PREFIX + "k2")+
+                " = "+getId(Consts.RIO_PREFIX + "miter2")+".next();\n");
+      key.genCompareTo(cb, getId(Consts.RIO_PREFIX + "k1"), 
+          getId(Consts.RIO_PREFIX + "k2"));
+      cb.append("if (" + Consts.RIO_PREFIX + "ret != 0) { return " + 
+          Consts.RIO_PREFIX + "ret; }\n");
       cb.append("}\n");
-      cb.append("ret = ("+getId("set1")+".size() - "+getId("set2")+".size());\n");
+      cb.append(Consts.RIO_PREFIX + "ret = ("+getId(Consts.RIO_PREFIX + "set1")+
+          ".size() - "+getId(Consts.RIO_PREFIX + "set2")+".size());\n");
       cb.append("}\n");
     }
     
@@ -77,14 +97,20 @@
       }
       cb.append("{\n");
       incrLevel();
-      cb.append("org.apache.hadoop.record.Index "+getId("midx")+" = a.startMap(\""+tag+"\");\n");
+      cb.append("org.apache.hadoop.record.Index " + 
+          getId(Consts.RIO_PREFIX + "midx")+" = " + 
+          Consts.RECORD_INPUT + ".startMap(\""+tag+"\");\n");
       cb.append(fname+"=new "+getType()+"();\n");
-      cb.append("for (; !"+getId("midx")+".done(); "+getId("midx")+".incr()) {\n");
-      key.genReadMethod(cb, getId("k"),getId("k"), true);
-      value.genReadMethod(cb, getId("v"), getId("v"), true);
-      cb.append(fname+".put("+getId("k")+","+getId("v")+");\n");
+      cb.append("for (; !"+getId(Consts.RIO_PREFIX + "midx")+".done(); "+
+          getId(Consts.RIO_PREFIX + "midx")+".incr()) {\n");
+      key.genReadMethod(cb, getId(Consts.RIO_PREFIX + "k"),
+          getId(Consts.RIO_PREFIX + "k"), true);
+      value.genReadMethod(cb, getId(Consts.RIO_PREFIX + "v"), 
+          getId(Consts.RIO_PREFIX + "v"), true);
+      cb.append(fname+".put("+getId(Consts.RIO_PREFIX + "k")+","+
+          getId(Consts.RIO_PREFIX + "v")+");\n");
       cb.append("}\n");
-      cb.append("a.endMap(\""+tag+"\");\n");
+      cb.append(Consts.RECORD_INPUT + ".endMap(\""+tag+"\");\n");
       decrLevel();
       cb.append("}\n");
     }
@@ -98,16 +124,24 @@
         key.getWrapperType()+","+value.getWrapperType()+">> ";
       cb.append("{\n");
       incrLevel();
-      cb.append("a.startMap("+fname+",\""+tag+"\");\n");
-      cb.append(setType+getId("es")+" = "+fname+".entrySet();\n");
-      cb.append("for("+iterType+getId("midx")+" = "+getId("es")+".iterator(); "+getId("midx")+".hasNext();) {\n");
-      cb.append(entryType+getId("me")+" = "+getId("midx")+".next();\n");
-      cb.append(key.getType()+" "+getId("k")+" = "+getId("me")+".getKey();\n");
-      cb.append(value.getType()+" "+getId("v")+" = "+getId("me")+".getValue();\n");
-      key.genWriteMethod(cb, getId("k"), getId("k"));
-      value.genWriteMethod(cb, getId("v"), getId("v"));
+      cb.append(Consts.RECORD_OUTPUT + ".startMap("+fname+",\""+tag+"\");\n");
+      cb.append(setType+getId(Consts.RIO_PREFIX + "es")+" = "+
+          fname+".entrySet();\n");
+      cb.append("for("+iterType+getId(Consts.RIO_PREFIX + "midx")+" = "+
+          getId(Consts.RIO_PREFIX + "es")+".iterator(); "+
+          getId(Consts.RIO_PREFIX + "midx")+".hasNext();) {\n");
+      cb.append(entryType+getId(Consts.RIO_PREFIX + "me")+" = "+
+          getId(Consts.RIO_PREFIX + "midx")+".next();\n");
+      cb.append(key.getType()+" "+getId(Consts.RIO_PREFIX + "k")+" = "+
+          getId(Consts.RIO_PREFIX + "me")+".getKey();\n");
+      cb.append(value.getType()+" "+getId(Consts.RIO_PREFIX + "v")+" = "+
+          getId(Consts.RIO_PREFIX + "me")+".getValue();\n");
+      key.genWriteMethod(cb, getId(Consts.RIO_PREFIX + "k"), 
+          getId(Consts.RIO_PREFIX + "k"));
+      value.genWriteMethod(cb, getId(Consts.RIO_PREFIX + "v"), 
+          getId(Consts.RIO_PREFIX + "v"));
       cb.append("}\n");
-      cb.append("a.endMap("+fname+",\""+tag+"\");\n");
+      cb.append(Consts.RECORD_OUTPUT + ".endMap("+fname+",\""+tag+"\");\n");
       cb.append("}\n");
       decrLevel();
     }
@@ -156,11 +190,34 @@
     }
   }
   
+  class CppMap extends CppCompType {
+    
+    JType.CppType key;
+    JType.CppType value;
+    
+    CppMap(JType.CppType key, JType.CppType value) {
+      super("::std::map< "+key.getType()+", "+ value.getType()+" >");
+      this.key = key;
+      this.value = value;
+    }
+    
+    String getTypeIDObjectString() {
+      return "new ::hadoop::MapTypeID(" + 
+        key.getTypeIDObjectString() + ", " + 
+        value.getTypeIDObjectString() + ")";
+    }
+
+    void genSetRTIFilter(CodeBuffer cb) {
+      key.genSetRTIFilter(cb);
+      value.genSetRTIFilter(cb);
+    }
+
+  }
+  
   /** Creates a new instance of JMap */
   public JMap(JType t1, JType t2) {
     setJavaType(new JavaMap(t1.getJavaType(), t2.getJavaType()));
-    setCppType(new CppCompType(" ::std::map<"+t1.getCppType().getType()+","+
-                               t2.getCppType().getType()+">"));
+    setCppType(new CppMap(t1.getCppType(), t2.getCppType()));
     setCType(new CType());
     keyType = t1;
     valueType = t2;



Mime
View raw message