Author: tomwhite
Date: Wed Mar 14 14:36:16 2007
New Revision: 518338
URL: http://svn.apache.org/viewvc?view=rev&rev=518338
Log:
HADOOP-1089. Make the C++ version of write and read v-int agree with the Java versions.
Contributed by Milind Bhandarkar.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/src/c++/librecordio/Makefile
lucene/hadoop/trunk/src/c++/librecordio/archive.hh
lucene/hadoop/trunk/src/c++/librecordio/binarchive.cc
lucene/hadoop/trunk/src/c++/librecordio/csvarchive.cc
lucene/hadoop/trunk/src/c++/librecordio/exception.cc
lucene/hadoop/trunk/src/c++/librecordio/exception.hh
lucene/hadoop/trunk/src/c++/librecordio/recordio.hh
lucene/hadoop/trunk/src/c++/librecordio/test/Makefile
lucene/hadoop/trunk/src/c++/librecordio/test/test.cc
lucene/hadoop/trunk/src/c++/librecordio/test/testFromJava.cc
lucene/hadoop/trunk/src/c++/librecordio/xmlarchive.cc
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Wed Mar 14 14:36:16 2007
@@ -62,6 +62,10 @@
restarted it consumes 80% CPU. (Dhruba Borthakur via
tomwhite)
+19. HADOOP-1089. Make the C++ version of write and read v-int
+ agree with the Java versions. (Milind Bhandarkar via
+ tomwhite)
+
Release 0.12.0 - 2007-03-02
Modified: lucene/hadoop/trunk/src/c++/librecordio/Makefile
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/Makefile?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/Makefile (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/Makefile Wed Mar 14 14:36:16 2007
@@ -14,28 +14,30 @@
# limitations under the License.
#
+COPTS=-g3 -O0 -Wall
+
all: librecordio.a test
librecordio.a: recordio.o filestream.o binarchive.o csvarchive.o xmlarchive.o exception.o
ar cru librecordio.a recordio.o filestream.o binarchive.o csvarchive.o xmlarchive.o exception.o
recordio.o: recordio.cc recordio.hh archive.hh
- g++ -g3 -O0 -c -I${XERCESCROOT}/include -o recordio.o recordio.cc
+ g++ ${COPTS} -c -I${XERCESCROOT}/include -o recordio.o recordio.cc
filestream.o: filestream.cc recordio.hh filestream.hh
- g++ -g3 -O0 -c -o filestream.o filestream.cc
+ g++ ${COPTS} -c -o filestream.o filestream.cc
binarchive.o: binarchive.cc recordio.hh binarchive.hh archive.hh
- g++ -g3 -O0 -c -o binarchive.o binarchive.cc
+ g++ ${COPTS} -c -o binarchive.o binarchive.cc
csvarchive.o: csvarchive.cc recordio.hh csvarchive.hh archive.hh
- g++ -g3 -O0 -c -o csvarchive.o csvarchive.cc
+ g++ ${COPTS} -c -o csvarchive.o csvarchive.cc
xmlarchive.o: xmlarchive.cc recordio.hh xmlarchive.hh archive.hh
- g++ -g3 -O0 -c -I${XERCESCROOT}/include -o xmlarchive.o xmlarchive.cc
+ g++ ${COPTS} -c -I${XERCESCROOT}/include -o xmlarchive.o xmlarchive.cc
exception.o: exception.cc exception.hh
- g++ -g3 -O0 -c -o exception.o exception.cc
+ g++ ${COPTS} -c -o exception.o exception.cc
recordio.cc: recordio.hh archive.hh exception.hh
filestream.cc: recordio.hh filestream.hh
Modified: lucene/hadoop/trunk/src/c++/librecordio/archive.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/archive.hh?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/archive.hh (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/archive.hh Wed Mar 14 14:36:16 2007
@@ -72,6 +72,7 @@
}
endMap(idx, tag);
}
+ virtual ~IArchive() {}
};
class OArchive {
@@ -115,6 +116,7 @@
}
endMap(v.size(), tag);
}
+ virtual ~OArchive() {}
};
}; // end namespace hadoop
#endif /*ARCHIVE_HH_*/
Modified: lucene/hadoop/trunk/src/c++/librecordio/binarchive.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/binarchive.cc?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/binarchive.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/binarchive.cc Wed Mar 14 14:36:16 2007
@@ -17,6 +17,7 @@
*/
#include "binarchive.hh"
+#include <rpc/types.h>
#include <rpc/xdr.h>
@@ -38,63 +39,6 @@
}
}
-static void serializeInt(int32_t t, OutStream& stream)
-{
- if (t >= -120 && t <= 127) {
- int8_t b = t;
- stream.write(&b, 1);
- return;
- }
-
- int8_t len = -120;
- if (t < 0) {
- t ^= 0xFFFFFFFF; // take one's complement
- len = -124;
- }
-
- uint32_t tmp = t;
- while (tmp != 0) {
- tmp = tmp >> 8;
- len--;
- }
-
- stream.write(&len, 1);
- len = (len < -124) ? -(len + 124) : -(len + 120);
-
- for (uint32_t idx = len; idx != 0; idx--) {
- uint32_t shiftbits = (idx - 1) * 8;
- uint32_t mask = 0xFF << shiftbits;
- uint8_t b = (t & mask) >> shiftbits;
- stream.write(&b, 1);
- }
-}
-
-static void deserializeInt(int32_t& t, InStream& stream)
-{
- int8_t b;
- if (1 != stream.read(&b, 1)) {
- throw new IOException("Error deserializing int");
- }
- if (b >= -120) {
- t = b;
- return;
- }
- bool isNegative = (b < -124);
- b = isNegative ? -(b + 124) : -(b + 120);
- uint8_t barr[b];
- if (b != stream.read(barr, b)) {
- throw new IOException("Error deserializing int");
- }
- t = 0;
- for (int idx = 0; idx < b; idx++) {
- t = t << 8;
- t |= (barr[idx] & 0xFF);
- }
- if (isNegative) {
- t ^= 0xFFFFFFFF;
- }
-}
-
static void serializeLong(int64_t t, OutStream& stream)
{
if (t >= -112 && t <= 127) {
@@ -105,7 +49,7 @@
int8_t len = -112;
if (t < 0) {
- t &= 0xFFFFFFFFFFFFFFFFLL; // take one's complement
+ t ^= 0xFFFFFFFFFFFFFFFFLL; // take one's complement
len = -120;
}
@@ -149,10 +93,23 @@
t |= (barr[idx] & 0xFF);
}
if (isNegative) {
- t ^= 0xFFFFFFFFFFFFFFFFL;
+ t ^= 0xFFFFFFFFFFFFFFFFLL;
}
}
+static void serializeInt(int32_t t, OutStream& stream)
+{
+ int64_t longVal = t;
+ ::serializeLong(longVal, stream);
+}
+
+static void deserializeInt(int32_t& t, InStream& stream)
+{
+ int64_t longVal;
+ ::deserializeLong(longVal, stream);
+ t = longVal;
+}
+
static void serializeFloat(float t, OutStream& stream)
{
char buf[sizeof(float)];
@@ -223,7 +180,9 @@
void hadoop::IBinArchive::deserialize(int32_t& t, const char* tag)
{
- ::deserializeInt(t, stream);
+ int64_t longVal = 0LL;
+ ::deserializeLong(longVal, stream);
+ t = longVal;
}
void hadoop::IBinArchive::deserialize(int64_t& t, const char* tag)
@@ -302,7 +261,8 @@
void hadoop::OBinArchive::serialize(int32_t t, const char* tag)
{
- ::serializeInt(t, stream);
+ int64_t longVal = t;
+ ::serializeLong(longVal, stream);
}
void hadoop::OBinArchive::serialize(int64_t t, const char* tag)
Modified: lucene/hadoop/trunk/src/c++/librecordio/csvarchive.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/csvarchive.cc?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/csvarchive.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/csvarchive.cc Wed Mar 14 14:36:16 2007
@@ -114,7 +114,7 @@
if (len%2 == 1) { // len is guaranteed to be even
throw new IOException("Errror deserializing buffer.");
}
- len >> 1;
+ len = len >> 1;
for (size_t idx = 0; idx < len; idx++) {
char buf[3];
buf[0] = s[2*idx];
@@ -298,7 +298,7 @@
{
printCommaUnlessFirst();
stream.write("#",1);
- for(int idx = 0; idx < len; idx++) {
+ for(size_t idx = 0; idx < len; idx++) {
uint8_t b = t[idx];
char sval[3];
sprintf(sval,"%2x",b);
Modified: lucene/hadoop/trunk/src/c++/librecordio/exception.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/exception.cc?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/exception.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/exception.cc Wed Mar 14 14:36:16 2007
@@ -17,7 +17,9 @@
*/
#include "exception.hh"
+#ifdef USE_EXECINFO
#include <execinfo.h>
+#endif
#include <errno.h>
#include <sstream>
@@ -42,7 +44,11 @@
mReason(reason)
{
+#ifdef USE_EXECINFO
mCalls = backtrace(mCallStack, sMaxCallStackDepth);
+#else
+ mCalls = 0;
+#endif
}
/**
@@ -81,7 +87,9 @@
if (mLocation.size() != 0) {
stream << " thrown at " << mLocation << "\n";
}
+#ifdef USE_EXECINFO
printCallStack(stream);
+#endif
if (mReason) {
stream << "caused by: ";
mReason->print(stream);
@@ -98,6 +106,7 @@
return stream.str();
}
+#ifdef USE_EXECINFO
/**
* Print the call stack where the exception was created.
*/
@@ -114,6 +123,7 @@
}
free(symbols);
}
+#endif
const char* Exception::getTypename() const {
return "Exception";
Modified: lucene/hadoop/trunk/src/c++/librecordio/exception.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/exception.hh?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/exception.hh (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/exception.hh Wed Mar 14 14:36:16 2007
@@ -68,10 +68,12 @@
*/
virtual std::string toString() const;
+#ifdef USE_EXECINFO
/**
* Print the call stack where the exception was created.
*/
virtual void printCallStack(std::ostream& stream=std::cerr) const;
+#endif
const std::string& getMessage() const {
return mMessage;
Modified: lucene/hadoop/trunk/src/c++/librecordio/recordio.hh
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/recordio.hh?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/recordio.hh (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/recordio.hh Wed Mar 14 14:36:16 2007
@@ -33,11 +33,13 @@
class InStream {
public:
virtual ssize_t read(void *buf, size_t buflen) = 0;
+ virtual ~InStream() {}
};
class OutStream {
public:
virtual ssize_t write(const void *buf, size_t len) = 0;
+ virtual ~OutStream() {}
};
class IArchive;
@@ -45,11 +47,11 @@
class Record {
public:
- virtual bool validate() const = 0;
virtual void serialize(OArchive& archive, const char* tag) const = 0;
virtual void deserialize(IArchive& archive, const char* tag) = 0;
virtual const std::string& type() const = 0;
virtual const std::string& signature() const = 0;
+ virtual ~Record() {}
};
enum RecFormat { kBinary, kXML, kCSV };
Modified: lucene/hadoop/trunk/src/c++/librecordio/test/Makefile
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/test/Makefile?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/test/Makefile (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/test/Makefile Wed Mar 14 14:36:16 2007
@@ -14,23 +14,25 @@
# limitations under the License.
#
+COPTS=-g3 -O0 -Wall
+
all: test testFromJava
test: test.o test.jr.o
g++ -g3 -O0 -o test test.o test.jr.o -L.. -L${XERCESCROOT}/lib -lrecordio -lxerces-c
test.o: test.cc
- g++ -g3 -O0 -c -I.. -o test.o test.cc
+ g++ ${COPTS} -c -I.. -o test.o test.cc
testFromJava: testFromJava.o test.jr.o
g++ -g3 -O0 -o testFromJava testFromJava.o test.jr.o -L.. -L${XERCESCROOT}/lib -lrecordio
-lxerces-c
testFromJava.o: testFromJava.cc
- g++ -g3 -O0 -c -I.. -o testFromJava.o testFromJava.cc
+ g++ ${COPTS} -c -I.. -o testFromJava.o testFromJava.cc
test.jr.o: test.jr.cc
- g++ -g3 -O0 -c -I.. -o test.jr.o test.jr.cc
+ g++ ${COPTS} -c -I.. -o test.jr.o test.jr.cc
%.jr.cc %.jr.hh: %.jr
${HADOOP_HOME}/bin/rcc --language c++ $<
Modified: lucene/hadoop/trunk/src/c++/librecordio/test/test.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/test/test.cc?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/test/test.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/test/test.cc Wed Mar 14 14:36:16 2007
@@ -34,9 +34,6 @@
r1.setLongVal(0x5a5a5a5a5a5aLL);
std::string& s = r1.getStringVal();
s = "random text";
- std::string& buf = r1.getBufferVal();
- std::vector<std::string>& v = r1.getVectorVal();
- std::map<std::string,std::string>& m = r1.getMapVal();
writer.write(r1);
ostream.close();
hadoop::FileInStream istream;
@@ -62,9 +59,6 @@
r1.setLongVal(0x5a5a5a5a5a5aLL);
std::string& s = r1.getStringVal();
s = "random text";
- std::string& buf = r1.getBufferVal();
- std::vector<std::string>& v = r1.getVectorVal();
- std::map<std::string,std::string>& m = r1.getMapVal();
writer.write(r1);
ostream.close();
hadoop::FileInStream istream;
@@ -90,9 +84,6 @@
r1.setLongVal(0x5a5a5a5a5a5aLL);
std::string& s = r1.getStringVal();
s = "random text";
- std::string& buf = r1.getBufferVal();
- std::vector<std::string>& v = r1.getVectorVal();
- std::map<std::string,std::string>& m = r1.getMapVal();
writer.write(r1);
ostream.close();
hadoop::FileInStream istream;
Modified: lucene/hadoop/trunk/src/c++/librecordio/test/testFromJava.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/test/testFromJava.cc?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/test/testFromJava.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/test/testFromJava.cc Wed Mar 14 14:36:16 2007
@@ -30,9 +30,6 @@
r1.setLongVal(0x5a5a5a5a5a5aLL);
std::string& s = r1.getStringVal();
s = "random text";
- std::string& buf = r1.getBufferVal();
- std::vector<std::string>& v = r1.getVectorVal();
- std::map<std::string,std::string>& m = r1.getMapVal();
{
hadoop::FileInStream istream;
istream.open("/tmp/hadooptemp.dat");
Modified: lucene/hadoop/trunk/src/c++/librecordio/xmlarchive.cc
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/c%2B%2B/librecordio/xmlarchive.cc?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/c++/librecordio/xmlarchive.cc (original)
+++ lucene/hadoop/trunk/src/c++/librecordio/xmlarchive.cc Wed Mar 14 14:36:16 2007
@@ -164,7 +164,7 @@
if (len%2 == 1) { // len is guaranteed to be even
throw new IOException("Errror deserializing buffer.");
}
- len >> 1;
+ len = len >> 1;
std::string t;
for (size_t idx = 0; idx < len; idx++) {
char buf[3];
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/CppGenerator.java Wed Mar
14 14:36:16 2007
@@ -46,20 +46,21 @@
FileWriter cc = new FileWriter(name+".cc");
FileWriter hh = new FileWriter(name+".hh");
- hh.write("#ifndef __"+name.toUpperCase().replace('.','_')+"__\n");
- hh.write("#define __"+name.toUpperCase().replace('.','_')+"__\n");
+ String fileName = (new File(name)).getName();
+ hh.write("#ifndef __"+fileName.toUpperCase().replace('.','_')+"__\n");
+ hh.write("#define __"+fileName.toUpperCase().replace('.','_')+"__\n");
hh.write("#include \"recordio.hh\"\n");
for (Iterator<JFile> iter = ilist.iterator(); iter.hasNext();) {
hh.write("#include \""+iter.next().getName()+".hh\"\n");
}
- cc.write("#include \""+name+".hh\"\n");
+ cc.write("#include \""+fileName+".hh\"\n");
for (Iterator<JRecord> iter = rlist.iterator(); iter.hasNext();) {
iter.next().genCppCode(hh, cc, options);
}
- hh.write("#endif //"+name.toUpperCase().replace('.','_')+"__\n");
+ hh.write("#endif //"+fileName.toUpperCase().replace('.','_')+"__\n");
hh.close();
cc.close();
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java?view=diff&rev=518338&r1=518337&r2=518338
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/record/compiler/JMap.java Wed Mar 14 14:36:16
2007
@@ -161,7 +161,7 @@
/** Creates a new instance of JMap */
public JMap(JType t1, JType t2) {
setJavaType(new JavaMap(t1.getJavaType(), t2.getJavaType()));
- setCppType(new CppType(" ::std::map<"+t1.getCppType().getType()+","+
+ setCppType(new CppCompType(" ::std::map<"+t1.getCppType().getType()+","+
t2.getCppType().getType()+">"));
setCType(new CType());
keyType = t1;
|