Return-Path: Delivered-To: apmail-avro-commits-archive@www.apache.org Received: (qmail 52824 invoked from network); 27 Mar 2011 12:21:39 -0000 Received: from hermes.apache.org (HELO mail.apache.org) (140.211.11.3) by minotaur.apache.org with SMTP; 27 Mar 2011 12:21:39 -0000 Received: (qmail 18338 invoked by uid 500); 27 Mar 2011 12:21:39 -0000 Delivered-To: apmail-avro-commits-archive@avro.apache.org Received: (qmail 18325 invoked by uid 500); 27 Mar 2011 12:21:38 -0000 Mailing-List: contact commits-help@avro.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@avro.apache.org Delivered-To: mailing list commits@avro.apache.org Received: (qmail 18318 invoked by uid 99); 27 Mar 2011 12:21:38 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 27 Mar 2011 12:21:38 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sun, 27 Mar 2011 12:21:31 +0000 Received: by eris.apache.org (Postfix, from userid 65534) id 9841623889FA; Sun, 27 Mar 2011 12:21:08 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1085921 - in /avro/trunk: ./ lang/c++/ lang/c++/api/ lang/c++/impl/ lang/c++/jsonschemas/ lang/c++/test/ Date: Sun, 27 Mar 2011 12:21:08 -0000 To: commits@avro.apache.org From: thiru@apache.org X-Mailer: svnmailer-1.0.8 Message-Id: <20110327122108.9841623889FA@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: thiru Date: Sun Mar 27 12:21:07 2011 New Revision: 1085921 URL: http://svn.apache.org/viewvc?rev=1085921&view=rev Log: AVRO-783. Specifc object support in C++ Added: avro/trunk/lang/c++/api/Specific.hh avro/trunk/lang/c++/impl/avrogencpp.cc avro/trunk/lang/c++/jsonschemas/union_array_union avro/trunk/lang/c++/jsonschemas/union_map_union avro/trunk/lang/c++/test/AvrogencppTests.cc avro/trunk/lang/c++/test/SpecificTests.cc Modified: avro/trunk/CHANGES.txt avro/trunk/lang/c++/CMakeLists.txt avro/trunk/lang/c++/api/Generic.hh avro/trunk/lang/c++/build.sh avro/trunk/lang/c++/impl/Generic.cc Modified: avro/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1085921&r1=1085920&r2=1085921&view=diff ============================================================================== --- avro/trunk/CHANGES.txt (original) +++ avro/trunk/CHANGES.txt Sun Mar 27 12:21:07 2011 @@ -239,6 +239,8 @@ Avro 1.5.0 (10 March 2011) AVRO-781. Generic data support in C++. (thiru) + AVRO-783. Specifc object support in C++. (thiru) + BUG FIXES AVRO-764. Java: Bug in BinaryData.compare() with offset comparison. Modified: avro/trunk/lang/c++/CMakeLists.txt URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/CMakeLists.txt?rev=1085921&r1=1085920&r2=1085921&view=diff ============================================================================== --- avro/trunk/lang/c++/CMakeLists.txt (original) +++ avro/trunk/lang/c++/CMakeLists.txt Sun Mar 27 12:21:07 2011 @@ -34,7 +34,7 @@ set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${BU project (Avro-cpp) -find_package (Boost 1.38 COMPONENTS regex filesystem system) +find_package (Boost 1.38 COMPONENTS regex filesystem system program_options) include_directories (api ${BUILD_DIRECTORY}) @@ -82,10 +82,31 @@ add_custom_target (testgen2 add_custom_command (OUTPUT ${BUILD_DIRECTORY}/AvroYacc.cc COMMAND bison --defines=AvroYacc.hh -o AvroYacc.cc ../parser/AvroYacc.yy WORKING_DIRECTORY ${BUILD_DIRECTORY}) + add_custom_command (OUTPUT ${BUILD_DIRECTORY}/AvroLex.cc COMMAND flex -oAvroLex.cc ../parser/AvroLex.ll WORKING_DIRECTORY ${BUILD_DIRECTORY}) +add_custom_target (bigrecord_hh + COMMAND avrogencpp -i jsonschemas/bigrecord + -o ${BUILD_DIRECTORY}/bigrecord.hh -n testgen + DEPENDS avrogencpp) + +add_custom_target (bigrecord2_hh + COMMAND avrogencpp -i jsonschemas/bigrecord2 + -o ${BUILD_DIRECTORY}/bigrecord2.hh -n testgen2 + DEPENDS avrogencpp) + +add_custom_target (union_array_union_hh + COMMAND avrogencpp -i jsonschemas/union_array_union + -o ${BUILD_DIRECTORY}/union_array_union.hh -n uau + DEPENDS avrogencpp) + +add_custom_target (union_map_union_hh + COMMAND avrogencpp -i jsonschemas/union_map_union + -o ${BUILD_DIRECTORY}/union_map_union.hh -n umu + DEPENDS avrogencpp) + macro (test name) add_executable (${name} test/${name}.cc) target_link_libraries (${name} avrocpp ${Boost_LIBRARIES}) @@ -100,10 +121,21 @@ target_link_libraries (CodecTests avrocp add_executable (StreamTests test/StreamTests.cc) target_link_libraries (StreamTests avrocpp ${Boost_LIBRARIES}) +add_executable (SpecificTests test/SpecificTests.cc) +target_link_libraries (SpecificTests avrocpp ${Boost_LIBRARIES}) + +add_executable (avrogencpp impl/avrogencpp.cc) +target_link_libraries (avrogencpp avrocpp ${Boost_LIBRARIES}) + add_executable (testgentest test/testgen.cc) add_dependencies (testgentest testgen testgen2) target_link_libraries (testgentest avrocpp ${Boost_LIBRARIES}) +add_executable (AvrogencppTests test/AvrogencppTests.cc) +add_dependencies (AvrogencppTests bigrecord_hh bigrecord2_hh + union_array_union_hh union_map_union_hh) +target_link_libraries (AvrogencppTests avrocpp ${BOOST_LIBRARIES}) + include (InstallRequiredSystemLibraries) set (CPACK_PACKAGE_FILE_NAME "avrocpp-${AVRO_VERSION_MAJOR}") Modified: avro/trunk/lang/c++/api/Generic.hh URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Generic.hh?rev=1085921&r1=1085920&r2=1085921&view=diff ============================================================================== --- avro/trunk/lang/c++/api/Generic.hh (original) +++ avro/trunk/lang/c++/api/Generic.hh Sun Mar 27 12:21:07 2011 @@ -23,8 +23,8 @@ #include #include -#include #include +#include #include "Node.hh" #include "Types.hh" @@ -237,6 +237,11 @@ public: const ValidSchema& readerSchema, const DecoderPtr& decoder); void read(GenericDatum& datum) const; + + /** + * Reads a generic datum from the stream, using the given schema. + */ + static void read(Decoder& d, GenericDatum& g, const ValidSchema& s); }; @@ -249,6 +254,24 @@ public: GenericWriter(const ValidSchema& s, const EncoderPtr& encoder); void write(const GenericDatum& datum) const; + + /** + * Writes a generic datum on to the stream, using the given schema. + */ + static void write(Encoder& e, const GenericDatum& g, const ValidSchema& s); +}; + +template struct codec_traits; + +template <> struct codec_traits > { + static void encode(Encoder& e, + const std::pair& p) { + GenericWriter::write(e, p.second, p.first); + } + + static void decode(Decoder& d, std::pair& p) { + GenericReader::read(d, p.second, p.first); + } }; } // namespace avro #endif Added: avro/trunk/lang/c++/api/Specific.hh URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Specific.hh?rev=1085921&view=auto ============================================================================== --- avro/trunk/lang/c++/api/Specific.hh (added) +++ avro/trunk/lang/c++/api/Specific.hh Sun Mar 27 12:21:07 2011 @@ -0,0 +1,207 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef avro_Codec_hh__ +#define avro_Codec_hh__ + +#include +#include +#include +#include + +#include "boost/array.hpp" + +#include "Encoder.hh" +#include "Decoder.hh" + +/** + * A bunch of templates and specializations for encoding and decoding + * specific types. + * + * Primitive AVRO types BOOLEAN, INT, LONG, FLOAT, DOUBLE, STRING and BYTES + * get decoded to and encoded from C++ types bool, int32_t, int64_t, float, + * double, std::string and std::vector respectively. In addition, + * std::vector for aribtrary type T gets encoded as an Avro array of T. + * Similarly, std::map for arbitrary type T gets encoded + * as an Avro map with value type T. + * + * Users can have their custom types encoded/decoded by specializing + * avro::codec_traits class for their types. + */ +namespace avro { + +template void encode(Encoder& e, const T& t); +template void decode(Decoder& d, T& t); + +template +struct codec_traits { +}; + +template <> struct codec_traits { + static void encode(Encoder& e, bool b) { + e.encodeBool(b); + } + + static void decode(Decoder& d, bool& b) { + b = d.decodeBool(); + } +}; + +template <> struct codec_traits { + static void encode(Encoder& e, int32_t i) { + e.encodeInt(i); + } + + static void decode(Decoder& d, int32_t& i) { + i = d.decodeInt(); + } +}; + +template <> struct codec_traits { + static void encode(Encoder& e, int64_t l) { + e.encodeLong(l); + } + + static void decode(Decoder& d, int64_t& l) { + l = d.decodeLong(); + } +}; + +template <> struct codec_traits { + static void encode(Encoder& e, float f) { + e.encodeFloat(f); + } + + static void decode(Decoder& d, float& f) { + f = d.decodeFloat(); + } +}; + +template <> struct codec_traits { + static void encode(Encoder& e, double d) { + e.encodeDouble(d); + } + + static void decode(Decoder& d, double& dbl) { + dbl = d.decodeDouble(); + } +}; + +template <> struct codec_traits { + static void encode(Encoder& e, const std::string& s) { + e.encodeString(s); + } + + static void decode(Decoder& d, std::string& s) { + s = d.decodeString(); + } +}; + +template <> struct codec_traits > { + static void encode(Encoder& e, const std::vector& b) { + e.encodeBytes(b); + } + + static void decode(Decoder& d, std::vector& s) { + d.decodeBytes(s); + } +}; + +template struct codec_traits > { + static void encode(Encoder& e, const boost::array& b) { + e.encodeFixed(&b[0], N); + } + + static void decode(Decoder& d, boost::array& s) { + std::vector v(N); + d.decodeFixed(N, v); + std::copy(&v[0], &v[0] + N, &s[0]); + } +}; + +template struct codec_traits > { + static void encode(Encoder& e, const std::vector& b) { + e.arrayStart(); + if (! b.empty()) { + e.setItemCount(b.size()); + for (typename std::vector::const_iterator it = b.begin(); + it != b.end(); ++it) { + e.startItem(); + avro::encode(e, *it); + } + } + e.arrayEnd(); + } + + static void decode(Decoder& d, std::vector& s) { + s.clear(); + for (size_t n = d.arrayStart(); n != 0; n = d.arrayNext()) { + for (size_t i = 0; i < n; ++i) { + T t; + avro::decode(d, t); + s.push_back(t); + } + } + } +}; + +template struct codec_traits > { + static void encode(Encoder& e, const std::map& b) { + e.mapStart(); + if (! b.empty()) { + e.setItemCount(b.size()); + for (typename std::map::const_iterator + it = b.begin(); + it != b.end(); ++it) { + e.startItem(); + avro::encode(e, it->first); + avro::encode(e, it->second); + } + } + e.mapEnd(); + } + + static void decode(Decoder& d, std::map& s) { + s.clear(); + for (size_t n = d.mapStart(); n != 0; n = d.mapNext()) { + for (size_t i = 0; i < n; ++i) { + std::string k; + avro::decode(d, k); + T t; + avro::decode(d, t); + s[k] = t; + } + } + } +}; + +template +void encode(Encoder& e, const T& t) { + codec_traits::encode(e, t); +} + +template +void decode(Decoder& d, T& t) { + codec_traits::decode(d, t); +} + +} // namespace avro +#endif // avro_Codec_hh__ + + + Modified: avro/trunk/lang/c++/build.sh URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/build.sh?rev=1085921&r1=1085920&r2=1085921&view=diff ============================================================================== --- avro/trunk/lang/c++/build.sh (original) +++ avro/trunk/lang/c++/build.sh Sun Mar 27 12:21:07 2011 @@ -82,6 +82,8 @@ case "$target" in ./build/testgentest ./build/CodecTests ./build/StreamTests + ./build/SpecificTests + ./build/AvrogencppTests ;; dist) Modified: avro/trunk/lang/c++/impl/Generic.cc URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/Generic.cc?rev=1085921&r1=1085920&r2=1085921&view=diff ============================================================================== --- avro/trunk/lang/c++/impl/Generic.cc (original) +++ avro/trunk/lang/c++/impl/Generic.cc Sun Mar 27 12:21:07 2011 @@ -93,7 +93,8 @@ GenericRecord::GenericRecord(const NodeP } GenericReader::GenericReader(const ValidSchema& s, const DecoderPtr& decoder) : - schema_(s), isResolving_(false), decoder_(decoder) + schema_(s), isResolving_(dynamic_cast(&(*decoder)) != 0), + decoder_(decoder) { } @@ -245,6 +246,11 @@ void GenericReader::read(GenericDatum& d } } +void GenericReader::read(Decoder& d, GenericDatum& g, const ValidSchema& s) +{ + read(g, s.root(), d, dynamic_cast(&d) != 0); +} + static void typeMismatch(Type t, Type u) { throw Exception(boost::format("Type mismatch %1% v %2%") % @@ -422,4 +428,10 @@ void GenericWriter::write(const GenericD } } +void GenericWriter::write(Encoder& e, const GenericDatum& g, + const ValidSchema& s) +{ + write(g, s.root(), e); +} + } // namespace avro Added: avro/trunk/lang/c++/impl/avrogencpp.cc URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/avrogencpp.cc?rev=1085921&view=auto ============================================================================== --- avro/trunk/lang/c++/impl/avrogencpp.cc (added) +++ avro/trunk/lang/c++/impl/avrogencpp.cc Sun Mar 27 12:21:07 2011 @@ -0,0 +1,533 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include + +#include "Compiler.hh" +#include "ValidSchema.hh" +#include "NodeImpl.hh" + +using std::ostream; +using std::ifstream; +using std::ofstream; +using std::string; +using std::vector; +using avro::NodePtr; +using avro::resolveSymbol; + +using boost::lexical_cast; + +using avro::ValidSchema; +using avro::compileJsonSchema; + +class CodeGen { + size_t unionNumber_; + std::ostream& os_; + const std::string ns_; + const std::string headerFile_; + const std::string schemaFile_; + boost::mt19937 random_; + + std::string guard(); + std::string fullname(const string& name) const; + std::string generateEnumType(const NodePtr& n); + std::string cppTypeOf(const NodePtr& n); + std::string generateRecordType(const NodePtr& n); + std::string unionName(); + std::string generateUnionType(const NodePtr& n); + std::string generateType(const NodePtr& n); + void generateEnumTraits(const NodePtr& n); + void generateTraits(const NodePtr& n); + void generateRecordTraits(const NodePtr& n); + void generateUnionTraits(const NodePtr& n); + void emitCopyright(); +public: + CodeGen(std::ostream& os, std::string& ns, + std::string& schemaFile, std::string& headerFile) : + unionNumber_(0), os_(os), ns_(ns), + schemaFile_(schemaFile), headerFile_(headerFile), + random_(::time(0)) { } + void generate(const ValidSchema& schema); +}; + +string CodeGen::fullname(const string& name) const +{ + return ns_.empty() ? name : (ns_ + "::" + name); +} + +string CodeGen::generateEnumType(const NodePtr& n) +{ + os_ << "enum " << n->name() << " {\n"; + size_t c = n->names(); + for (int i = 0; i < c; ++i) { + os_ << " " << n->nameAt(i) << ",\n"; + } + os_ << "};\n\n"; + return n->name(); +} + +string CodeGen::cppTypeOf(const NodePtr& n) +{ + switch (n->type()) { + case avro::AVRO_STRING: + return "std::string"; + case avro::AVRO_BYTES: + return "std::vector"; + case avro::AVRO_INT: + return "int32_t"; + case avro::AVRO_LONG: + return "int64_t"; + case avro::AVRO_FLOAT: + return "float"; + case avro::AVRO_DOUBLE: + return "double"; + case avro::AVRO_BOOL: + return "bool"; + case avro::AVRO_RECORD: + case avro::AVRO_ENUM: + return fullname(n->name()); + case avro::AVRO_ARRAY: + return "std::vector<" + cppTypeOf(n->leafAt(0)) + " >"; + case avro::AVRO_MAP: + return "std::mapleafAt(1)) + " >"; + case avro::AVRO_FIXED: + return "boost::array(n->fixedSize()) + ">"; + case avro::AVRO_SYMBOLIC: + return cppTypeOf(resolveSymbol(n)); + default: + return "$Undefined$"; + } +} + +static string cppNameOf(const NodePtr& n) +{ + switch (n->type()) { + case avro::AVRO_NULL: + return "null"; + case avro::AVRO_STRING: + return "string"; + case avro::AVRO_BYTES: + return "bytes"; + case avro::AVRO_INT: + return "int"; + case avro::AVRO_LONG: + return "long"; + case avro::AVRO_FLOAT: + return "float"; + case avro::AVRO_DOUBLE: + return "double"; + case avro::AVRO_BOOL: + return "bool"; + case avro::AVRO_RECORD: + case avro::AVRO_ENUM: + case avro::AVRO_FIXED: + return n->name(); + case avro::AVRO_ARRAY: + return "array"; + case avro::AVRO_MAP: + return "map"; + case avro::AVRO_SYMBOLIC: + return cppNameOf(resolveSymbol(n)); + default: + return "$Undefined$"; + } +} + +string CodeGen::generateRecordType(const NodePtr& n) +{ + size_t c = n->leaves(); + vector types; + for (int i = 0; i < c; ++i) { + types.push_back(generateType(n->leafAt(i))); + } + + os_ << "struct " << n->name() << " {\n"; + for (int i = 0; i < c; ++i) { + os_ << " " << types[i] + << " " << n->nameAt(i) << ";\n"; + } + os_ << "};\n\n"; + return n->name(); +} + +void makeCanonical(string& s, bool foldCase) +{ + for (string::iterator it = s.begin(); it != s.end(); ++it) { + if (isalpha(*it)) { + if (foldCase) { + *it = toupper(*it); + } + } else if (! isdigit(*it)) { + *it = '_'; + } + } +} + +string CodeGen::unionName() +{ + string s = schemaFile_; + string::size_type n = s.find_last_of("/\\"); + if (n != string::npos) { + s = s.substr(n); + } + makeCanonical(s, false); + + return s + "_Union__" + boost::lexical_cast(unionNumber_++) + "__"; +} + +string CodeGen::generateUnionType(const NodePtr& n) +{ + size_t c = n->leaves(); + vector types; + vector names; + for (size_t i = 0; i < c; ++i) { + const NodePtr& nn = n->leafAt(i); + types.push_back(generateType(nn)); + names.push_back(cppNameOf(nn)); + } + + string result = unionName(); + os_ << "struct " << result << " {\n" + << "private:\n" + << " size_t idx_;\n" + << " boost::any value_;\n" + << "public:\n" + << " size_t idx() const { return idx_; }\n"; + + for (size_t i = 0; i < c; ++i) { + const NodePtr& nn = n->leafAt(i); + if (nn->type() == avro::AVRO_NULL) { + os_ << " void set_null() {\n" + << " idx_ = " << i << ";\n" + << " value_ = boost::any();\n" + << " }\n"; + continue; + } + string type = types[i]; + string name = names[i]; + os_ << " " << type << " get_" << name << "() const {\n" + << " if (idx_ != " << i << ") {\n" + << " throw avro::Exception(\"Invalid type for " + << "union\");\n" + << " }\n" + << " return boost::any_cast<" << type << " >(value_);\n" + << " }\n"; + + os_ << " void set_" << name << "(const " << type << "& v) {\n" + << " idx_ = " << i << ";\n" + << " value_ = v;\n" + << " }\n"; + } + os_ << " " << result << "() : idx_(0) {\n"; + if (n->leafAt(0)->type() != avro::AVRO_NULL) { + os_ << " value_ = " << types[0] << "();\n"; + } + os_ << " }\n"; + os_ << "};\n\n"; + + return result; +} + +string CodeGen::generateType(const NodePtr& n) +{ + switch (n->type()) { + case avro::AVRO_STRING: + case avro::AVRO_BYTES: + case avro::AVRO_INT: + case avro::AVRO_LONG: + case avro::AVRO_FLOAT: + case avro::AVRO_DOUBLE: + case avro::AVRO_BOOL: + case avro::AVRO_NULL: + case avro::AVRO_FIXED: + return cppTypeOf(n); + case avro::AVRO_ARRAY: + case avro::AVRO_MAP: + generateType(n->leafAt(n->type() == avro::AVRO_ARRAY ? 0 : 1)); + return cppTypeOf(n); + case avro::AVRO_RECORD: + return generateRecordType(n); + case avro::AVRO_ENUM: + return generateEnumType(n); + case avro::AVRO_UNION: + return generateUnionType(n); + case avro::AVRO_SYMBOLIC: + return cppTypeOf(resolveSymbol(n)); + } + return "$Undefuned$"; +} + +void CodeGen::generateEnumTraits(const NodePtr& n) +{ + string fn = fullname(n->name()); + os_ << "template<> struct codec_traits<" << fn << "> {\n" + << " static void encode(Encoder& e, " << fn << " v) {\n" + << " e.encodeEnum(v);\n" + << " }\n" + << " static void decode(Decoder& d, " << fn << "& v) {\n" + << " v = static_cast<" << fn << ">(d.decodeEnum());\n" + << " }\n" + << "};\n\n"; +} + +void CodeGen::generateRecordTraits(const NodePtr& n) +{ + size_t c = n->leaves(); + for (int i = 0; i < c; ++i) { + generateTraits(n->leafAt(i)); + } + + string fn = fullname(n->name()); + os_ << "template<> struct codec_traits<" << fn << "> {\n" + << " static void encode(Encoder& e, const " << fn << "& v) {\n"; + + for (size_t i = 0; i < c; ++i) { + os_ << " avro::encode(e, v." << n->nameAt(i) << ");\n"; + } + + os_ << " }\n" + << " static void decode(Decoder& d, " << fn << "& v) {\n"; + + for (size_t i = 0; i < c; ++i) { + os_ << " avro::decode(d, v." << n->nameAt(i) << ");\n"; + } + + os_ << " }\n" + << "};\n\n"; +} + +void CodeGen::generateUnionTraits(const NodePtr& n) +{ + size_t c = n->leaves(); + + for (size_t i = 0; i < c; ++i) { + const NodePtr& nn = n->leafAt(i); + generateTraits(nn); + } + + string name = unionName(); + string fn = fullname(name); + + os_ << "template<> struct codec_traits<" << fn << "> {\n" + << " static void encode(Encoder& e, " << fn << " v) {\n" + << " e.encodeUnionIndex(v.idx());\n" + << " switch (v.idx()) {\n"; + + for (size_t i = 0; i < c; ++i) { + const NodePtr& nn = n->leafAt(i); + os_ << " case " << i << ":\n"; + if (nn->type() == avro::AVRO_NULL) { + os_ << " e.encodeNull();\n"; + } else { + os_ << " avro::encode(e, v.get_" << cppNameOf(nn) + << "());\n"; + } + os_ << " break;\n"; + } + + os_ << " }\n" + << " }\n" + << " static void decode(Decoder& d, " << fn << "& v) {\n" + << " size_t n = d.decodeUnionIndex();\n" + << " if (n >= " << c << ") { throw avro::Exception(\"" + "Union index too big\"); }\n" + << " switch (n) {\n"; + + for (size_t i = 0; i < c; ++i) { + const NodePtr& nn = n->leafAt(i); + os_ << " case " << i << ":\n"; + if (nn->type() == avro::AVRO_NULL) { + os_ << " d.decodeNull();\n" + << " v.set_null();\n"; + } else { + os_ << " {\n" + << " " << cppTypeOf(nn) << " vv;\n" + << " avro::decode(d, vv);\n" + << " v.set_" << cppNameOf(nn) << "(vv);\n" + << " }\n"; + } + os_ << " break;\n"; + } + os_ << " }\n" + << " }\n" + << "};\n\n"; +} + +void CodeGen::generateTraits(const NodePtr& n) +{ + switch (n->type()) { + case avro::AVRO_STRING: + case avro::AVRO_BYTES: + case avro::AVRO_INT: + case avro::AVRO_LONG: + case avro::AVRO_FLOAT: + case avro::AVRO_DOUBLE: + case avro::AVRO_BOOL: + case avro::AVRO_NULL: + break; + case avro::AVRO_RECORD: + generateRecordTraits(n); + break; + case avro::AVRO_ENUM: + generateEnumTraits(n); + break; + case avro::AVRO_ARRAY: + case avro::AVRO_MAP: + generateTraits(n->leafAt(n->type() == avro::AVRO_ARRAY ? 0 : 1)); + break; + case avro::AVRO_UNION: + generateUnionTraits(n); + break; + case avro::AVRO_FIXED: + break; + } +} + +void CodeGen::emitCopyright() +{ + os_ << + "/**\n" + " * Licensed to the Apache Software Foundation (ASF) under one\n" + " * or more contributor license agreements. See the NOTICE file\n" + " * distributed with this work for additional information\n" + " * regarding copyright ownership. The ASF licenses this file\n" + " * to you under the Apache License, Version 2.0 (the\n" + " * \"License\"); you may not use this file except in compliance\n" + " * with the License. You may obtain a copy of the License at\n" + " *\n" + " * http://www.apache.org/licenses/LICENSE-2.0\n" + " *\n" + " * Unless required by applicable law or agreed to in writing, " + "software\n" + " * distributed under the License is distributed on an " + "\"AS IS\" BASIS,\n" + " * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express " + "or implied.\n" + " * See the License for the specific language governing " + "permissions and\n" + " * limitations under the License.\n" + " */\n\n\n"; +} + +string CodeGen::guard() +{ + string h = headerFile_; + makeCanonical(h, true); + return h + "_" + lexical_cast(random_()) + "__H_"; +} + +void CodeGen::generate(const ValidSchema& schema) +{ + emitCopyright(); + + string h = guard(); + + os_ << "#ifndef " << h << "\n"; + os_ << "#define " << h << "\n\n\n"; + + os_ << "#include \"boost/any.hpp\"\n" + << "#include \"Specific.hh\"\n" + << "#include \"Encoder.hh\"\n" + << "#include \"Decoder.hh\"\n"; + + if (! ns_.empty()) { + os_ << "namespace " << ns_ << " {\n"; + } + + const NodePtr& root = schema.root(); + generateType(root); + + if (! ns_.empty()) { + os_ << "}\n"; + } + + os_ << "namespace avro {\n"; + + unionNumber_ = 0; + + generateTraits(root); + + os_ << "}\n"; + + os_ << "#endif\n"; + os_.flush(); +} + +namespace po = boost::program_options; + +string NS("namespace"); +string OUT("output"); +string IN("input"); + +int main(int argc, char** argv) +{ + po::options_description desc("Allowed options"); + desc.add_options() + ("help,h", "produce help message") + ("namespace,n", po::value(), "set namespace for generated code") + ("input,i", po::value(), "input file") + ("output,o", po::value(), "output file to generate"); + + po::variables_map vm; + po::store(po::parse_command_line(argc, argv, desc), vm); + po::notify(vm); + + + if (vm.count("help") || vm.count(IN) == 0 || vm.count(OUT) == 0) { + std::cout << desc << std::endl; + return 1; + } + + string ns = vm.count(NS) > 0 ? vm[NS].as() : string(); + string outf = vm[OUT].as(); + string inf = vm[IN].as(); + try { + ValidSchema schema; + + if (! inf.empty()) { + ifstream in(inf.c_str()); + compileJsonSchema(in, schema); + } else { + compileJsonSchema(std::cin, schema); + } + + if (! outf.empty()) { + ofstream out(outf.c_str()); + CodeGen(out, ns, inf, outf).generate(schema); + } else { + CodeGen(std::cout, ns, inf, outf).generate(schema); + } + return 0; + } catch (std::exception &e) { + std::cerr << "Failed to parse or compile schema: " + << e.what() << std::endl; + return 1; + } + +} Added: avro/trunk/lang/c++/jsonschemas/union_array_union URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/jsonschemas/union_array_union?rev=1085921&view=auto ============================================================================== --- avro/trunk/lang/c++/jsonschemas/union_array_union (added) +++ avro/trunk/lang/c++/jsonschemas/union_array_union Sun Mar 27 12:21:07 2011 @@ -0,0 +1,8 @@ +{ + "type": "record", + "name": "r1", + "fields" : [ + {"name": "id", "type": "string"}, + {"name": "val", "type": [{"type": "array", "items": {"name": "r3", "type": "record", "fields": [{"name": "name", "type": "string"}, {"name": "data", "type": "bytes"}, {"name": "rev", "type": ["string", "null"]}]}}, "null"]} + ] +} Added: avro/trunk/lang/c++/jsonschemas/union_map_union URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/jsonschemas/union_map_union?rev=1085921&view=auto ============================================================================== --- avro/trunk/lang/c++/jsonschemas/union_map_union (added) +++ avro/trunk/lang/c++/jsonschemas/union_map_union Sun Mar 27 12:21:07 2011 @@ -0,0 +1,8 @@ +{ + "type": "record", + "name": "r1", + "fields" : [ + {"name": "id", "type": "string"}, + {"name": "val", "type": [{"type": "map", "values": {"name": "r3", "type": "record", "fields": [{"name": "name", "type": "string"}, {"name": "data", "type": "bytes"}, {"name": "rev", "type": ["string", "null"]}]}}, "null"]} + ] +} Added: avro/trunk/lang/c++/test/AvrogencppTests.cc URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/test/AvrogencppTests.cc?rev=1085921&view=auto ============================================================================== --- avro/trunk/lang/c++/test/AvrogencppTests.cc (added) +++ avro/trunk/lang/c++/test/AvrogencppTests.cc Sun Mar 27 12:21:07 2011 @@ -0,0 +1,191 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "bigrecord.hh" +#include "bigrecord2.hh" +#include "union_array_union.hh" +#include "union_map_union.hh" +#include "Compiler.hh" + +#include +#include + +using std::auto_ptr; +using std::map; +using std::string; +using std::vector; +using std::ifstream; + +using avro::ValidSchema; +using avro::OutputStream; +using avro::InputStream; +using avro::Encoder; +using avro::Decoder; +using avro::EncoderPtr; +using avro::DecoderPtr; +using avro::memoryInputStream; +using avro::memoryOutputStream; +using avro::binaryEncoder; +using avro::validatingEncoder; +using avro::binaryDecoder; +using avro::validatingDecoder; + +void setRecord(testgen::RootRecord &myRecord) +{ + uint8_t fixed[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + myRecord.mylong = 212; + myRecord.nestedrecord.inval1 = std::numeric_limits::min(); + myRecord.nestedrecord.inval2 = "hello world"; + myRecord.nestedrecord.inval3 = std::numeric_limits::max(); + + myRecord.mymap["one"] = 100; + myRecord.mymap["two"] = 200; + + myRecord.myarray.push_back(3434.9); + myRecord.myarray.push_back(7343.9); + myRecord.myarray.push_back(-63445.9); + myRecord.myenum = testgen::one; + + map m; + m["one"] = 1; + m["two"] = 2; + myRecord.myunion.set_map(m); + + vector v; + v.push_back(1); + v.push_back(2); + myRecord.anotherunion.set_bytes(v); + + myRecord.mybool = true; + myRecord.anothernested.inval1 = std::numeric_limits::max(); + myRecord.anothernested.inval2 = "goodbye world"; + myRecord.anothernested.inval3 = std::numeric_limits::min(); + memcpy(&myRecord.myfixed[0], fixed, myRecord.myfixed.size()); + myRecord.anotherint = 4534; + myRecord.bytes.push_back(10); + myRecord.bytes.push_back(20); +} + +void check(const testgen::RootRecord& r1, const testgen::RootRecord& r2) +{ + BOOST_CHECK_EQUAL(r1.mylong, r2.mylong); + BOOST_CHECK_EQUAL(r1.nestedrecord.inval1, r2.nestedrecord.inval1); + BOOST_CHECK_EQUAL(r1.nestedrecord.inval2, r2.nestedrecord.inval2); + BOOST_CHECK_EQUAL(r1.nestedrecord.inval3, r2.nestedrecord.inval3); + BOOST_CHECK(r1.mymap == r2.mymap); + BOOST_CHECK(r1.myarray == r2.myarray); + BOOST_CHECK_EQUAL(r1.myunion.idx(), r2.myunion.idx()); + BOOST_CHECK(r1.myunion.get_map() == r2.myunion.get_map()); + BOOST_CHECK_EQUAL(r1.anotherunion.idx(), r2.anotherunion.idx()); + BOOST_CHECK(r1.anotherunion.get_bytes() == r2.anotherunion.get_bytes()); + BOOST_CHECK_EQUAL(r1.mybool, r2.mybool); + BOOST_CHECK_EQUAL(r1.anothernested.inval1, r2.anothernested.inval1); + BOOST_CHECK_EQUAL(r1.anothernested.inval2, r2.anothernested.inval2); + BOOST_CHECK_EQUAL(r1.anothernested.inval3, r2.anothernested.inval3); + BOOST_CHECK_EQUAL_COLLECTIONS(r1.myfixed.begin(), r1.myfixed.end(), + r2.myfixed.begin(), r2.myfixed.end()); + BOOST_CHECK_EQUAL(r1.anotherint, r2.anotherint); + BOOST_CHECK_EQUAL_COLLECTIONS(r1.bytes.begin(), r1.bytes.end(), + r2.bytes.begin(), r2.bytes.end()); +} + +void testEncoding() +{ + ValidSchema s; + ifstream ifs("jsonschemas/bigrecord"); + compileJsonSchema(ifs, s); + auto_ptr os = memoryOutputStream(); + EncoderPtr e = validatingEncoder(s, binaryEncoder()); + e->init(*os); + testgen::RootRecord t1; + setRecord(t1); + avro::encode(*e, t1); + e->flush(); + + DecoderPtr d = validatingDecoder(s, binaryDecoder()); + auto_ptr is = memoryInputStream(*os); + d->init(*is); + testgen::RootRecord t2; + avro::decode(*d, t2); + + check(t2, t1); +} + +void setRecord(uau::r1& r) +{ +} + +void check(const uau::r1& r1, const uau::r1& r2) +{ + +} + +void setRecord(umu::r1& r) +{ +} + +void check(const umu::r1& r1, const umu::r1& r2) +{ + +} + +template struct schemaFilename { }; +template <> struct schemaFilename { + static const char value[]; +}; +const char schemaFilename::value[] = "jsonschemas/union_array_union"; +template <> struct schemaFilename { + static const char value[]; +}; +const char schemaFilename::value[] = "jsonschemas/union_map_union"; + +template +void testEncoding2() +{ + ValidSchema s; + ifstream ifs(schemaFilename::value); + compileJsonSchema(ifs, s); + + auto_ptr os = memoryOutputStream(); + EncoderPtr e = validatingEncoder(s, binaryEncoder()); + e->init(*os); + T t1; + setRecord(t1); + avro::encode(*e, t1); + e->flush(); + + DecoderPtr d = validatingDecoder(s, binaryDecoder()); + auto_ptr is = memoryInputStream(*os); + d->init(*is); + T t2; + avro::decode(*d, t2); + + check(t2, t1); +} + +boost::unit_test::test_suite* +init_unit_test_suite(int argc, char* argv[]) +{ + boost::unit_test::test_suite* ts = BOOST_TEST_SUITE("Code generator tests"); + ts->add(BOOST_TEST_CASE(testEncoding)); + ts->add(BOOST_TEST_CASE(testEncoding2)); + ts->add(BOOST_TEST_CASE(testEncoding2)); + return ts; +} + Added: avro/trunk/lang/c++/test/SpecificTests.cc URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/test/SpecificTests.cc?rev=1085921&view=auto ============================================================================== --- avro/trunk/lang/c++/test/SpecificTests.cc (added) +++ avro/trunk/lang/c++/test/SpecificTests.cc Sun Mar 27 12:21:07 2011 @@ -0,0 +1,200 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include + +#include "Specific.hh" +#include "Stream.hh" + +using std::auto_ptr; +using std::string; +using std::vector; +using std::map; +using boost::array; + +namespace avro { + +class C { + int32_t i_; + int64_t l_; +public: + C() : i_(0), l_(0) { } + C(int32_t i, int64_t l) : i_(i), l_(l) { } + int32_t i() const { return i_; } + int64_t l() const { return l_; } + void i(int32_t ii) { i_ = ii; } + void l(int64_t ll) { l_ = ll; } + + bool operator==(const C& oth) const { + return i_ == oth.i_ && l_ == oth.l_; + } +}; + +template <> struct codec_traits { + static void encode(Encoder& e, const C& c) { + e.encodeInt(c.i()); + e.encodeLong(c.l()); + } + + static void decode(Decoder& d, C& c) { + c.i(d.decodeInt()); + c.l(d.decodeLong()); + } +}; + +namespace specific { + +class Test { + auto_ptr os; + EncoderPtr e; + DecoderPtr d; +public: + Test() : os(memoryOutputStream()), e(binaryEncoder()), d(binaryDecoder()) { + e->init(*os); + } + + template void encode(const T& t) { + avro::encode(*e, t); + e->flush(); + } + + template void decode(T& t) { + auto_ptr is = memoryInputStream(*os); + d->init(*is); + avro::decode(*d, t); + } +}; + +template T encodeAndDecode(const T& t) +{ + Test tst; + + tst.encode(t); + + T actual = T(); + + tst.decode(actual); + return actual; +} + +void testBool() +{ + bool b = encodeAndDecode(true); + BOOST_CHECK_EQUAL(b, true); +} + +void testInt() +{ + int32_t n = 10; + int32_t b = encodeAndDecode(n); + BOOST_CHECK_EQUAL(b, n); +} + +void testLong() +{ + int64_t n = -109; + int64_t b = encodeAndDecode(n); + BOOST_CHECK_EQUAL(b, n); +} + +void testFloat() +{ + float n = 10.19; + float b = encodeAndDecode(n); + BOOST_CHECK_CLOSE(b, n, 0.00001); +} + +void testDouble() +{ + double n = 10.00001; + double b = encodeAndDecode(n); + BOOST_CHECK_CLOSE(b, n, 0.00000001); +} + +void testString() +{ + string n = "abc"; + string b = encodeAndDecode(n); + BOOST_CHECK_EQUAL(b, n); +} + +void testBytes() +{ + uint8_t values[] = { 1, 7, 23, 47, 83 }; + vector n(values, values + 5); + vector b = encodeAndDecode(n); + BOOST_CHECK_EQUAL_COLLECTIONS(b.begin(), b.end(), n.begin(), n.end()); +} + +void testFixed() +{ + array n = { 1, 7, 23, 47, 83 }; + array b = encodeAndDecode(n); + BOOST_CHECK_EQUAL_COLLECTIONS(b.begin(), b.end(), n.begin(), n.end()); +} + +void testArray() +{ + int32_t values[] = { 101, 709, 409, 34 }; + vector n(values, values + 4); + vector b = encodeAndDecode(n); + + BOOST_CHECK_EQUAL_COLLECTIONS(b.begin(), b.end(), n.begin(), n.end()); +} + +void testMap() +{ + map n; + n["a"] = 1; + n["b"] = 101; + + map b = encodeAndDecode(n); + + BOOST_CHECK(b == n); +} + +void testCustom() +{ + C n(10, 1023); + C b = encodeAndDecode(n); + BOOST_CHECK(b == n); +} + +} +} + +boost::unit_test::test_suite* +init_unit_test_suite( int argc, char* argv[] ) +{ + using namespace boost::unit_test; + + test_suite* ts= BOOST_TEST_SUITE("Specific tests"); + ts->add(BOOST_TEST_CASE(avro::specific::testBool)); + ts->add(BOOST_TEST_CASE(avro::specific::testInt)); + ts->add(BOOST_TEST_CASE(avro::specific::testLong)); + ts->add(BOOST_TEST_CASE(avro::specific::testFloat)); + ts->add(BOOST_TEST_CASE(avro::specific::testDouble)); + ts->add(BOOST_TEST_CASE(avro::specific::testString)); + ts->add(BOOST_TEST_CASE(avro::specific::testBytes)); + ts->add(BOOST_TEST_CASE(avro::specific::testFixed)); + ts->add(BOOST_TEST_CASE(avro::specific::testArray)); + ts->add(BOOST_TEST_CASE(avro::specific::testMap)); + ts->add(BOOST_TEST_CASE(avro::specific::testCustom)); + return ts; +}