avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From th...@apache.org
Subject svn commit: r1072677 [1/3] - in /avro/trunk: ./ lang/c++/ lang/c++/api/ lang/c++/api/buffer/detail/ lang/c++/impl/ lang/c++/impl/parsing/ lang/c++/parser/ lang/c++/test/
Date Sun, 20 Feb 2011 18:23:19 GMT
Author: thiru
Date: Sun Feb 20 18:23:18 2011
New Revision: 1072677

URL: http://svn.apache.org/viewvc?rev=1072677&view=rev
Log:
AVRO-711. JSON encoder and decoder for C++

Added:
    avro/trunk/lang/c++/api/Decoder.hh
    avro/trunk/lang/c++/api/Encoder.hh
    avro/trunk/lang/c++/api/Stream.hh
    avro/trunk/lang/c++/impl/BinaryDecoder.cc
    avro/trunk/lang/c++/impl/BinaryEncoder.cc
    avro/trunk/lang/c++/impl/FileStream.cc
    avro/trunk/lang/c++/impl/Stream.cc
    avro/trunk/lang/c++/impl/parsing/
    avro/trunk/lang/c++/impl/parsing/JsonCodec.cc
    avro/trunk/lang/c++/impl/parsing/ResolvingDecoder.cc
    avro/trunk/lang/c++/impl/parsing/Symbol.cc
    avro/trunk/lang/c++/impl/parsing/Symbol.hh
    avro/trunk/lang/c++/impl/parsing/ValidatingCodec.cc
    avro/trunk/lang/c++/impl/parsing/ValidatingCodec.hh
    avro/trunk/lang/c++/test/CodecTests.cc
    avro/trunk/lang/c++/test/StreamTests.cc
Modified:
    avro/trunk/CHANGES.txt
    avro/trunk/lang/c++/.gitignore
    avro/trunk/lang/c++/CMakeLists.txt
    avro/trunk/lang/c++/README
    avro/trunk/lang/c++/api/NodeImpl.hh
    avro/trunk/lang/c++/api/Types.hh
    avro/trunk/lang/c++/api/ValidSchema.hh
    avro/trunk/lang/c++/api/buffer/detail/BufferDetail.hh
    avro/trunk/lang/c++/build.sh
    avro/trunk/lang/c++/impl/Types.cc
    avro/trunk/lang/c++/parser/AvroLex.ll

Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Sun Feb 20 18:23:18 2011
@@ -94,6 +94,8 @@ Avro 1.5.0 (unreleased)
     AVRO-730. Java: Add set() and remove() methods to GenericData.Array.
     (Chase Bradford via cutting)
 
+    AVRO-711. JSON encoder and decoder for C++.
+
   IMPROVEMENTS
 
     AVRO-765. Java: Improvement to BinaryDecoder readLong performance

Modified: avro/trunk/lang/c++/.gitignore
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/.gitignore?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/.gitignore (original)
+++ avro/trunk/lang/c++/.gitignore Sun Feb 20 18:23:18 2011
@@ -3,3 +3,13 @@ Makefile.in
 configure
 INSTALL
 aclocal.m4
+.svn/
+CMakeCache.txt
+CMakeFiles/
+CPackConfig.cmake
+CPackSourceConfig.cmake
+Makefile
+build/
+cmake_install.cmake
+m4/
+test.avro

Modified: avro/trunk/lang/c++/CMakeLists.txt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/CMakeLists.txt?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/CMakeLists.txt (original)
+++ avro/trunk/lang/c++/CMakeLists.txt Sun Feb 20 18:23:18 2011
@@ -16,7 +16,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-cmake_minimum_required (VERSION 2.8)
+cmake_minimum_required (VERSION 2.6)
 
 if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt)
     file(READ "${CMAKE_CURRENT_SOURCE_DIR}/VERSION.txt" AVRO_VERSION)
@@ -33,23 +33,29 @@ set (CMAKE_RUNTIME_OUTPUT_DIRECTORY ${BU
 set (CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${BUILD_DIRECTORY})
 
 project (Avro-cpp)
+
+find_package (Boost 1.38 COMPONENTS regex filesystem system)
+
 include_directories (api ${BUILD_DIRECTORY})
 add_library (avrocpp SHARED impl/Compiler.cc
         impl/Compiler.cc impl/CompilerNode.cc impl/Node.cc
         impl/NodeImpl.cc impl/Resolver.cc impl/ResolverSchema.cc impl/Schema.cc
         impl/Types.cc impl/Validator.cc impl/ValidSchema.cc impl/Zigzag.cc
+        impl/BinaryEncoder.cc impl/BinaryDecoder.cc
+        impl/Stream.cc impl/FileStream.cc
+        impl/parsing/Symbol.cc
+        impl/parsing/ValidatingCodec.cc
+        impl/parsing/JsonCodec.cc
+        impl/parsing/ResolvingDecoder.cc
         ${BUILD_DIRECTORY}/AvroYacc.cc ${BUILD_DIRECTORY}/AvroLex.cc)
-target_link_libraries (avrocpp boost_regex-mt)
+
+target_link_libraries (avrocpp ${Boost_LIBRARIES})
 
 add_executable (precompile test/precompile.cc)
 
 add_dependencies(avrocpp parser lexer)
 
-target_link_libraries (precompile avrocpp boost_regex-mt)
-
-find_package(BISON)
-
-find_package(FLEX)
+target_link_libraries (precompile avrocpp ${Boost_LIBRARIES})
 
 add_custom_command (OUTPUT ${BUILD_DIRECTORY}/bigrecord
     COMMAND precompile jsonschemas/bigrecord ${BUILD_DIRECTORY}/bigrecord
@@ -71,9 +77,13 @@ add_custom_target (testgen2
     WORKING_DIRECTORY ${BUILD_DIRECTORY}
     DEPENDS ${BUILD_DIRECTORY}/bigrecord2)
 
-BISON_TARGET (parser parser/AvroYacc.yy ${BUILD_DIRECTORY}/AvroYacc.cc)
-FLEX_TARGET (lexer parser/AvroLex.ll ${BUILD_DIRECTORY}/AvroLex.cc)
-
+add_custom_command (OUTPUT ${BUILD_DIRECTORY}/AvroYacc.cc
+    COMMAND bison --defines=AvroYacc.hh -o AvroYacc.cc ../parser/AvroYacc.yy
+    WORKING_DIRECTORY ${BUILD_DIRECTORY})
+add_custom_command (OUTPUT ${BUILD_DIRECTORY}/AvroLex.cc
+    COMMAND flex -oAvroLex.cc ../parser/AvroLex.ll
+    WORKING_DIRECTORY ${BUILD_DIRECTORY})
+    
 macro (test name)
     add_executable (${name} test/${name}.cc)
     target_link_libraries (${name} avrocpp boost_regex-mt)
@@ -82,6 +92,12 @@ endmacro (test)
 test(buffertest)
 test(unittest)
 
+add_executable (CodecTests test/CodecTests.cc)
+target_link_libraries (CodecTests avrocpp ${Boost_LIBRARIES})
+
+add_executable (StreamTests test/StreamTests.cc)
+target_link_libraries (StreamTests avrocpp ${Boost_LIBRARIES})
+
 add_executable (testgentest test/testgen.cc)
 add_dependencies (testgentest testgen testgen2)
 target_link_libraries (testgentest avrocpp boost_regex-mt)
@@ -93,9 +109,12 @@ set (CPACK_PACKAGE_FILE_NAME "avrocpp-${
 include (CPack)
 
 install(TARGETS avrocpp
-    LIBRARY DESTINATION lib
-    ARCHIVE DESTINATION lib
-    RUNTIME DESTINATION lib)
+    LIBRARY DESTINATION local/lib
+    ARCHIVE DESTINATION local/lib
+    RUNTIME DESTINATION local/lib)
 
-install(DIRECTORY api/ DESTINATION include/avro
+install(DIRECTORY api/ DESTINATION local/include/avro
     FILES_MATCHING PATTERN *.hh)
+
+set (CMAKE_BUILD_TYPE Release)
+

Modified: avro/trunk/lang/c++/README
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/README?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/README (original)
+++ avro/trunk/lang/c++/README Sun Feb 20 18:23:18 2011
@@ -45,6 +45,8 @@ If the Makefile is configured correctly,
     ./build/unittest
     ./build/buffertest
     ./build/testgentest
+    ./build/CodecTests
+    ./build/StreamTests
 
 To install
 

Added: avro/trunk/lang/c++/api/Decoder.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Decoder.hh?rev=1072677&view=auto
==============================================================================
--- avro/trunk/lang/c++/api/Decoder.hh (added)
+++ avro/trunk/lang/c++/api/Decoder.hh Sun Feb 20 18:23:18 2011
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef avro_Decoder_hh__
+#define avro_Decoder_hh__
+
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+#include "ValidSchema.hh"
+#include "Stream.hh"
+
+#include <boost/shared_ptr.hpp>
+
+/// \file
+///
+/// Low level support for decoding avro values.
+/// This class has two types of funtions.  One type of functions support
+/// decoding of leaf values (for example, decodeLong and
+/// decodeString). These functions have analogs in Encoder.
+/// 
+/// The other type of functions support decoding of maps and arrays.
+/// These functions are arrayStart, startItem, and arrayEnd
+/// (and similar functions for maps).
+
+namespace avro {
+
+class Decoder {
+public:
+    /// All future decoding will come from is, which should be valid
+    /// until replaced by another call to init() or this Decoder is
+    /// destructed.
+    virtual void init(InputStream& is) = 0;
+
+    /// Decodes a null from the current stream.
+    virtual void decodeNull() = 0;
+
+    /// Decodes a bool from the current stream
+    virtual bool decodeBool() = 0;
+
+    /// Decodes a 32-bit int from the current stream.
+    virtual int32_t decodeInt() = 0;
+
+    /// Decodes a 64-bit signed int from the current stream.
+    virtual int64_t decodeLong() = 0;
+
+    /// Decodes a single-precision floating point number from current stream.
+    virtual float decodeFloat() = 0;
+
+    /// Decodes a double-precision floating point number from current stream.
+    virtual double decodeDouble() = 0;
+
+    /// Decodes a UTF-8 string from the current stream.
+    std::string decodeString() {
+        std::string result;
+        decodeString(result);
+        return result;
+    }
+
+    /**
+     * Decodes a UTF-8 string from the stream and assigns it to value.
+     */
+    virtual void decodeString(std::string& value) = 0;
+
+    /// Skips a string on the current stream.
+    virtual void skipString() = 0;
+
+    /// Decodes arbitray binary data from the current stream.
+    std::vector<uint8_t> decodeBytes() {
+        std::vector<uint8_t> result;
+        decodeBytes(result);
+        return result;
+    }
+
+    /// Decodes arbitray binary data from the current stream and puts it
+    /// in value.
+    virtual void decodeBytes(std::vector<uint8_t>& value) = 0;
+
+    /// Skips bytes on the current stream.
+    virtual void skipBytes() = 0;
+
+    /// Decodes fixed length binary from the current stream.
+    std::vector<uint8_t> decodeFixed(size_t n) {
+        std::vector<uint8_t> result;
+        decodeFixed(n, result);
+        return result;
+    }
+
+    virtual void decodeFixed(size_t n, std::vector<uint8_t>& value) = 0;
+
+    /// Skips fixed length binary on the current stream.
+    virtual void skipFixed(size_t n) = 0;
+
+    /// Decodes enum from the current stream.
+    virtual size_t decodeEnum() = 0;
+
+    /// Start decoding an array. Returns the number of entries in first chunk.
+    virtual size_t arrayStart() = 0;
+
+    /// Returns the number of entries in next chunk. 0 if last.
+    virtual size_t arrayNext() = 0;
+
+    /// Tries to skip an array. If it can, it returns 0. Otherwise
+    /// it returns the number of elements to be skipped. The client
+    /// should skip the individual items. In such cases, skipArray
+    /// is identical to arrayStart.
+    virtual size_t skipArray() = 0;
+
+    /// Start decoding a map. Returns the number of entries in first chunk.
+    virtual size_t mapStart() = 0;
+
+    /// Returns the number of entries in next chunk. 0 if last.
+    virtual size_t mapNext() = 0;
+
+    /// Tries to skip a map. If it can, it returns 0. Otherwise
+    /// it returns the number of elements to be skipped. The client
+    /// should skip the individual items. In such cases, skipMap
+    /// is identical to mapStart.
+    virtual size_t skipMap() = 0;
+
+    /// Decodes a branch of a union. The actual value is to follow.
+    virtual size_t decodeUnionIndex() = 0;
+};
+
+typedef boost::shared_ptr<Decoder> DecoderPtr;
+
+class ResolvingDecoder : public Decoder {
+public:
+    /// Returns the order of fields for records.
+    /// The order of fields could be different from the order of their
+    /// order in the schema because the writer's field order could
+    /// be different. In order to avoid buffering and later use,
+    /// we return the values in the writer's field order.
+    virtual const std::vector<size_t>& fieldOrder() = 0;
+};
+
+typedef boost::shared_ptr<ResolvingDecoder> ResolvingDecoderPtr;
+/**
+ *  Returns an decoder that can decode binary Avro standard.
+ */
+DecoderPtr binaryDecoder();
+
+/**
+ *  Returns an decoder that validates sequence of calls to an underlying
+ *  Decoder against the given schema.
+ */
+DecoderPtr validatingDecoder(const ValidSchema& schema,
+    const DecoderPtr& base);
+
+/**
+ *  Returns an decoder that can decode Avro standard for JSON.
+ */
+DecoderPtr jsonDecoder(const ValidSchema& schema);
+
+/**
+ *  Returns a decoder that decodes avro data from base written according to
+ *  writerSchema and resolves against readerSchema.
+ *  The client uses the decoder as if the data were written using readerSchema.
+ *  // FIXME: Handle out of order fields.
+ */
+ResolvingDecoderPtr resolvingDecoder(const ValidSchema& writer,
+    const ValidSchema& reader, const DecoderPtr& base);
+
+
+}   // namespace avro
+
+#endif

Added: avro/trunk/lang/c++/api/Encoder.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Encoder.hh?rev=1072677&view=auto
==============================================================================
--- avro/trunk/lang/c++/api/Encoder.hh (added)
+++ avro/trunk/lang/c++/api/Encoder.hh Sun Feb 20 18:23:18 2011
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef avro_Encoder_hh__
+#define avro_Encoder_hh__
+
+#include <stdint.h>
+#include <string>
+#include <vector>
+
+#include "ValidSchema.hh"
+#include "Stream.hh"
+
+#include <boost/shared_ptr.hpp>
+
+/// \file
+///
+/// Low level support for encoding avro values.
+/// This class has two types of funtions.  One type of functions support
+/// the writing of leaf values (for example, encodeLong and
+/// encodeString).  These functions have analogs in Decoder.
+/// 
+/// The other type of functions support the writing of maps and arrays.
+/// These functions are arrayStart, startItem, and arrayEnd
+/// (and similar functions for maps).
+/// Some implementations of Encoder handle the
+/// buffering required to break large maps and arrays into blocks,
+/// which is necessary for applications that want to do streaming.
+
+namespace avro {
+
+class Encoder {
+public:
+    /// All future encodings will go to os, which should be valid until
+    /// it is reset with another call to init() or the encoder is
+    /// destructed.
+    virtual void init(OutputStream& os) = 0;
+
+    /// Flushes any data in internal buffers.
+    virtual void flush() = 0;
+
+    /// Encodes a null to the current stream.
+    virtual void encodeNull() = 0;
+
+    /// Encodes a bool to the current stream
+    virtual void encodeBool(bool b) = 0;
+
+    /// Encodes a 32-bit int to the current stream.
+    virtual void encodeInt(int32_t i) = 0;
+
+    /// Encodes a 64-bit signed int to the current stream.
+    virtual void encodeLong(int64_t l) = 0;
+
+    /// Encodes a single-precision floating point number to the current stream.
+    virtual void encodeFloat(float f) = 0;
+
+    /// Encodes a double-precision floating point number to the current stream.
+    virtual void encodeDouble(double d) = 0;
+
+    /// Encodes a UTF-8 string to the current stream.
+    virtual void encodeString(const std::string& s) = 0;
+
+    /// Encodes arbitray binary data to the current stream.
+    virtual void encodeBytes(const uint8_t *bytes, size_t len) = 0;
+
+    void encodeBytes(const std::vector<uint8_t>& bytes) {
+        encodeBytes(&bytes[0], bytes.size());
+    }
+
+    /// Encodes fixed length binary to the current stream.
+    virtual void encodeFixed(const uint8_t *bytes, size_t len) = 0;
+
+    void encodeFixed(const std::vector<uint8_t>& bytes) {
+        encodeFixed(&bytes[0], bytes.size());
+    }
+
+    /// Encodes enum to the current stream.
+    virtual void encodeEnum(size_t e) = 0;
+
+    /// Indicates that an array of items is being encoded.
+    virtual void arrayStart() = 0;
+
+    /// Indicates that the current array of items have ended.
+    virtual void arrayEnd() = 0;
+
+    /// Indicates that a map of items is being encoded.
+    virtual void mapStart() = 0;
+
+    /// Indicates that the current map of items have ended.
+    virtual void mapEnd() = 0;
+
+    /// Indicates that count number of items are to follow in the current array
+    /// or map.
+    virtual void setItemCount(size_t count) = 0;
+
+    /// Marks a beginning of an item in the current array or map.
+    virtual void startItem() = 0;
+
+    /// Encodes a branch of a union. The actual value is to follow.
+    virtual void encodeUnionIndex(size_t e) = 0;
+};
+
+typedef boost::shared_ptr<Encoder> EncoderPtr;
+
+/**
+ *  Returns an encoder that can encode binary Avro standard.
+ */
+EncoderPtr binaryEncoder();
+
+/**
+ *  Returns an encoder that validates sequence of calls to an underlying
+ *  Encoder against the given schema.
+ */
+EncoderPtr validatingEncoder(const ValidSchema& schema,
+    const EncoderPtr& base);
+
+/**
+ *  Returns an encoder that can encode Avro standard for JSON.
+ */
+EncoderPtr jsonEncoder(const ValidSchema& schema);
+
+}   // namespace avro
+
+#endif

Modified: avro/trunk/lang/c++/api/NodeImpl.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/NodeImpl.hh?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/NodeImpl.hh (original)
+++ avro/trunk/lang/c++/api/NodeImpl.hh Sun Feb 20 18:23:18 2011
@@ -342,7 +342,7 @@ class NodeUnion : public NodeImplUnion
     void printJson(std::ostream &os, int depth) const;
 
     bool isValid() const {
-        return (leafAttributes_.size() > 1);
+        return (leafAttributes_.size() >= 1);
     }
 };
 

Added: avro/trunk/lang/c++/api/Stream.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Stream.hh?rev=1072677&view=auto
==============================================================================
--- avro/trunk/lang/c++/api/Stream.hh (added)
+++ avro/trunk/lang/c++/api/Stream.hh Sun Feb 20 18:23:18 2011
@@ -0,0 +1,266 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef avro_Stream_hh__
+#define avro_Stream_hh__
+
+#include <memory>
+#include <string.h>
+#include <stdint.h>
+#include "boost/utility.hpp"
+#include "Exception.hh"
+
+namespace avro {
+class InputStream : boost::noncopyable {
+public:
+    InputStream() { }
+    virtual ~InputStream() { }
+
+    /**
+     * Returns some of available data.
+     *
+     * Returns true if some data is available, false if no more data is
+     * available or an error has occurred.
+     */
+    virtual bool next(const uint8_t** data, size_t* len) = 0;
+
+    /**
+     * "Returns" back some of the data to the stream. The returned
+     * data must be less than what was obtained in the last call to
+     * next().
+     */
+    virtual void backup(size_t len) = 0;
+
+    /**
+     * Skips number of bytes specified by len.
+     */
+    virtual void skip(size_t len) = 0;
+
+    /**
+     * Returns the number of bytes read from this stream so far.
+     * All the bytes made available through next are considered
+     * to be used unless, retutned back using backup.
+     */
+    virtual size_t byteCount() const = 0;
+};
+
+class OutputStream : boost::noncopyable {
+public:
+    OutputStream() { }
+    virtual ~OutputStream() { }
+
+    /**
+     * Returns a buffer that can be written into.
+     * On successful return, data has the pointer to the buffer
+     * and len has the number of bytes available at data.
+     */
+    virtual bool next(uint8_t** data, size_t* len) = 0;
+
+    /**
+     * "Returns" back to the stream some of the buffer obtained
+     * from in the last call to next().
+     */
+    virtual void backup(size_t len) = 0;
+
+    /**
+     * Number of bytes written so far into this stream. The whole buffer
+     * returned by next() is assumed to be written unless some of
+     * it was retutned using backup().
+     */
+    virtual uint64_t byteCount() const = 0;
+
+    /**
+     * Flushes any data remaining in the buffer to the stream's underlying
+     * store, if any.
+     */
+    virtual void flush() = 0;
+};
+
+/**
+ * Returns a new OutputStream, which grows in memory chunks of specified size.
+ */
+std::auto_ptr<OutputStream> memoryOutputStream(size_t chunkSize = 4 * 1024);
+
+/**
+ * Returns a new InputStream, with the data from the given byte array.
+ * It does not copy the data, the byte array should remain valid
+ * until the InputStream is used.
+ */
+std::auto_ptr<InputStream> memoryInputStream(const uint8_t* data, size_t len);
+
+/**
+ * Returns a new InputStream with the contents written into an
+ * outputstream. The output stream must have been returned by
+ * an earlier call to memoryOutputStream(). The contents for the new
+ * input stream are the snapshot of the outputstream. One can construct
+ * any number of memory input stream from a single memory output stream.
+ */
+std::auto_ptr<InputStream> memoryInputStream(const OutputStream& source);
+
+/**
+ * Returns a new OutputStream whose contents would be stored in a file.
+ * Data is written in chunks of given buffer size.
+ *
+ * If there is a file with the given name, it is truncated and overwritten.
+ * If there is no file with the given name, it is created.
+ */
+std::auto_ptr<OutputStream> fileOutputStream(const char* filename,
+    size_t bufferSize = 8 * 1024);
+
+/**
+ * Returns a new InputStream whose contents come from the given file.
+ * Data is read in chunks of given buffer size.
+ */
+std::auto_ptr<InputStream> fileInputStream(const char* filename,
+    size_t bufferSize = 8 * 1024);
+
+/** A convenience class for reading from an InputStream */
+struct StreamReader {
+    InputStream* in_;
+    const uint8_t* next_;
+    const uint8_t* end_;
+
+    StreamReader() : in_(0), next_(0), end_(0) { }
+
+    void reset(InputStream& is) {
+        if (in_ != 0) {
+            in_->backup(end_ - next_);
+        }
+        in_ = &is;
+        next_ = end_ = 0;
+    }
+
+    uint8_t read() {
+        if (next_ == end_) {
+            more();
+        }
+        return *next_++;
+    }
+
+    void readBytes(uint8_t* b, size_t n) {
+        while (n > 0) {
+            if (next_ == end_) {
+                more();
+            }
+            size_t q = end_ - next_;
+            if (q > n) {
+                q = n;
+            }
+            ::memcpy(b, next_, q);
+            next_ += q;
+            b += q;
+            n -= q;
+        }
+    }
+
+    void skipBytes(size_t n) {
+        if (n > (end_ - next_)) {
+            n -= end_ - next_;
+            next_ = end_;
+            in_->skip(n);
+        } else {
+            next_ += n;
+        }
+    }
+
+    bool fill() {
+        size_t n = 0;
+        while (in_->next(&next_, &n)) {
+            if (n != 0) {
+                end_ = next_ + n;
+                return true;
+            }
+        }
+        return false;
+    }
+
+    void more() {
+        if (! fill()) {
+            throw Exception("EOF reached");
+        }
+    }
+
+    bool hasMore() {
+        return (next_ == end_) ? fill() : true;
+    }
+};
+
+/**
+ * A convinience class to write data into an OutputStream.
+ */
+struct StreamWriter {
+    OutputStream* out_;
+    uint8_t* next_;
+    uint8_t* end_;
+
+    StreamWriter() : out_(0), next_(0), end_(0) { }
+
+    void reset(OutputStream& os) {
+        if (out_ != 0) {
+            out_->backup(end_ - next_);
+        }
+        out_ = &os;
+        next_ = end_;
+    }
+
+    void write(uint8_t c) {
+        if (next_ == end_) {
+            more();
+        }
+        *next_++ = c;
+    }
+
+    void writeBytes(const uint8_t* b, size_t n) {
+        while (n > 0) {
+            if (next_ == end_) {
+                more();
+            }
+            size_t q = end_ - next_;
+            if (q > n) {
+                q = n;
+            }
+            ::memcpy(next_, b, q);
+            next_ += q;
+            b += q;
+            n -= q;
+        }
+    }
+
+    void more() {
+        size_t n = 0;
+        while (out_->next(&next_, &n)) {
+            if (n != 0) {
+                end_ = next_ + n;
+                return;
+            }
+        }
+        throw Exception("EOF reached");
+    }
+
+    void flush() {
+        if (next_ != end_) {
+            out_->backup(end_ - next_);
+            next_ = end_;
+        }
+        out_->flush();
+    }
+};
+}   // namespace avro
+#endif
+
+

Modified: avro/trunk/lang/c++/api/Types.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/Types.hh?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/Types.hh (original)
+++ avro/trunk/lang/c++/api/Types.hh Sun Feb 20 18:23:18 2011
@@ -67,6 +67,8 @@ inline bool isAvroTypeOrPseudoType(Type 
 }
 
 
+const std::string& toString(Type type);
+
 std::ostream &operator<< (std::ostream &os, avro::Type type);
 
 /// define a type to identify Null in template functions

Modified: avro/trunk/lang/c++/api/ValidSchema.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/ValidSchema.hh?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/ValidSchema.hh (original)
+++ avro/trunk/lang/c++/api/ValidSchema.hh Sun Feb 20 18:23:18 2011
@@ -42,7 +42,7 @@ class ValidSchema 
   public:
 
     explicit ValidSchema(const Schema &schema);
-    explicit ValidSchema(const ValidSchema &schema);
+    ValidSchema(const ValidSchema &schema);
     ValidSchema();
 
     void setSchema(const Schema &schema);

Modified: avro/trunk/lang/c++/api/buffer/detail/BufferDetail.hh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/api/buffer/detail/BufferDetail.hh?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/api/buffer/detail/BufferDetail.hh (original)
+++ avro/trunk/lang/c++/api/buffer/detail/BufferDetail.hh Sun Feb 20 18:23:18 2011
@@ -23,6 +23,7 @@
 #include <boost/shared_array.hpp>
 #include <boost/static_assert.hpp>
 #include <boost/function.hpp>
+#include <boost/utility.hpp>
 #ifdef HAVE_BOOST_ASIO
 #include <boost/asio/buffer.hpp>
 #endif

Modified: avro/trunk/lang/c++/build.sh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/build.sh?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/build.sh (original)
+++ avro/trunk/lang/c++/build.sh Sun Feb 20 18:23:18 2011
@@ -80,6 +80,8 @@ case "$target" in
     ./build/buffertest
     ./build/unittest
     ./build/testgentest
+    ./build/CodecTests
+    ./build/StreamTests
 	;;
 
     dist)

Added: avro/trunk/lang/c++/impl/BinaryDecoder.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/BinaryDecoder.cc?rev=1072677&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/BinaryDecoder.cc (added)
+++ avro/trunk/lang/c++/impl/BinaryDecoder.cc Sun Feb 20 18:23:18 2011
@@ -0,0 +1,230 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define __STDC_LIMIT_MACROS
+
+#include "Decoder.hh"
+#include "Zigzag.hh"
+#include "Exception.hh"
+
+#include <boost/array.hpp>
+#include <boost/make_shared.hpp>
+
+namespace avro {
+
+using boost::make_shared;
+
+class BinaryDecoder : public Decoder {
+    StreamReader in_;
+    const uint8_t* next_;
+    const uint8_t* end_;
+
+    void init(InputStream& ib);
+    void decodeNull();
+    bool decodeBool();
+    int32_t decodeInt();
+    int64_t decodeLong();
+    float decodeFloat();
+    double decodeDouble();
+    void decodeString(std::string& value);
+    void skipString();
+    void decodeBytes(std::vector<uint8_t>& value);
+    void skipBytes();
+    void decodeFixed(size_t n, std::vector<uint8_t>& value);
+    void skipFixed(size_t n);
+    size_t decodeEnum();
+    size_t arrayStart();
+    size_t arrayNext();
+    size_t skipArray();
+    size_t mapStart();
+    size_t mapNext();
+    size_t skipMap();
+    size_t decodeUnionIndex();
+
+    int64_t doDecodeLong();
+    size_t doDecodeItemCount();
+    void more();
+};
+
+DecoderPtr binaryDecoder()
+{
+    return make_shared<BinaryDecoder>();
+}
+
+void BinaryDecoder::init(InputStream& is)
+{
+    in_.reset(is);
+}
+
+void BinaryDecoder::decodeNull()
+{
+}
+
+bool BinaryDecoder::decodeBool()
+{
+    uint8_t v = in_.read();
+    if (v == 0) {
+        return false;
+    } else if (v == 1) {
+        return true;
+    }
+    throw Exception("Invalid value for bool");
+}
+
+int32_t BinaryDecoder::decodeInt()
+{
+    int64_t val = doDecodeLong();
+    if (val < INT32_MIN || val > INT32_MAX) {
+        throw Exception(
+            boost::format("Value out of range for Avro int: %1%") % val);
+    }
+    return static_cast<int32_t>(val);
+}
+
+int64_t BinaryDecoder::decodeLong()
+{
+    return doDecodeLong();
+}
+
+float BinaryDecoder::decodeFloat()
+{
+    float result;
+    in_.readBytes(reinterpret_cast<uint8_t *>(&result), sizeof(float));
+    return result;
+}
+
+double BinaryDecoder::decodeDouble()
+{
+    double result;
+    in_.readBytes(reinterpret_cast<uint8_t *>(&result), sizeof(double));
+    return result;
+}
+
+void BinaryDecoder::decodeString(std::string& value)
+{
+    size_t len = decodeInt();
+    value.resize(len);
+    in_.readBytes(reinterpret_cast<uint8_t*>(&value[0]), len);
+}
+
+void BinaryDecoder::skipString()
+{
+    size_t len = decodeInt();
+    in_.skipBytes(len);
+}
+
+void BinaryDecoder::decodeBytes(std::vector<uint8_t>& value)
+{
+    size_t len = decodeInt();
+    value.resize(len);
+    in_.readBytes(&value[0], len);
+}
+
+void BinaryDecoder::skipBytes()
+{
+    size_t len = decodeInt();
+    in_.skipBytes(len);
+}
+
+void BinaryDecoder::decodeFixed(size_t n, std::vector<uint8_t>& value)
+{
+    value.resize(n);
+    in_.readBytes(&value[0], n);
+}
+
+void BinaryDecoder::skipFixed(size_t n)
+{
+    in_.skipBytes(n);
+}
+
+size_t BinaryDecoder::decodeEnum()
+{
+    return doDecodeLong();
+}
+
+size_t BinaryDecoder::arrayStart()
+{
+    return doDecodeItemCount();
+}
+
+size_t BinaryDecoder::doDecodeItemCount()
+{
+    int64_t result = doDecodeLong();
+    if (result < 0) {
+        doDecodeLong();
+        return -result;
+    }
+    return result;
+}
+
+size_t BinaryDecoder::arrayNext()
+{
+    return doDecodeLong();
+}
+
+size_t BinaryDecoder::skipArray()
+{
+    for (; ;) {
+        int64_t r = doDecodeLong();
+        if (r < 0) {
+            size_t n = doDecodeLong();            
+            in_.skipBytes(n);
+        } else {
+            return r;
+        }
+    }
+}
+
+size_t BinaryDecoder::mapStart()
+{
+    return doDecodeItemCount();
+}
+
+size_t BinaryDecoder::mapNext()
+{
+    return doDecodeItemCount();
+}
+
+size_t BinaryDecoder::skipMap()
+{
+    return skipArray();
+}
+
+size_t BinaryDecoder::decodeUnionIndex()
+{
+    return doDecodeLong();
+}
+
+int64_t BinaryDecoder::doDecodeLong() {
+    uint64_t encoded = 0;
+    int shift = 0;
+    uint8_t u;
+    do {
+        if (shift >= 64) {
+            throw Exception("Invalid Avro varint");
+        }
+        u = in_.read();
+        encoded |= static_cast<uint64_t>(u & 0x7f) << shift;
+        shift += 7;
+    } while (u & 0x80);
+
+    return decodeZigzag64(encoded);
+}
+
+}   // namespace avro
+

Added: avro/trunk/lang/c++/impl/BinaryEncoder.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/BinaryEncoder.cc?rev=1072677&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/BinaryEncoder.cc (added)
+++ avro/trunk/lang/c++/impl/BinaryEncoder.cc Sun Feb 20 18:23:18 2011
@@ -0,0 +1,166 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Encoder.hh"
+#include "Zigzag.hh"
+#include <boost/array.hpp>
+#include <boost/make_shared.hpp>
+
+namespace avro {
+
+using boost::make_shared;
+using boost::shared_ptr;
+
+class BinaryEncoder : public Encoder {
+    StreamWriter out_;
+    uint8_t *next_;
+    uint8_t *end_;
+
+    void init(OutputStream& os);
+    void flush();
+    void encodeNull();
+    void encodeBool(bool b);
+    void encodeInt(int32_t i);
+    void encodeLong(int64_t l);
+    void encodeFloat(float f);
+    void encodeDouble(double d);
+    void encodeString(const std::string& s);
+    void encodeBytes(const uint8_t *bytes, size_t len);
+    void encodeFixed(const uint8_t *bytes, size_t len);
+    void encodeEnum(size_t e);
+    void arrayStart();
+    void arrayEnd();
+    void mapStart();
+    void mapEnd();
+    void setItemCount(size_t count);
+    void startItem();
+    void encodeUnionIndex(size_t e);
+
+    void doEncodeLong(int64_t l);
+};
+
+EncoderPtr binaryEncoder()
+{
+    return make_shared<BinaryEncoder>();
+}
+
+void BinaryEncoder::init(OutputStream& os)
+{
+    out_.reset(os);
+}
+
+void BinaryEncoder::flush()
+{
+    out_.flush();
+}
+
+void BinaryEncoder::encodeNull()
+{
+}
+
+void BinaryEncoder::encodeBool(bool b)
+{
+    out_.write(b ? 1 : 0);
+}
+
+void BinaryEncoder::encodeInt(int32_t i)
+{
+    doEncodeLong(i);
+}
+
+void BinaryEncoder::encodeLong(int64_t l)
+{
+    doEncodeLong(l);
+}
+
+void BinaryEncoder::encodeFloat(float f)
+{
+    const uint8_t* p = reinterpret_cast<const uint8_t*>(&f);
+    out_.writeBytes(p, sizeof(float));
+}
+
+void BinaryEncoder::encodeDouble(double d)
+{
+    const uint8_t* p = reinterpret_cast<const uint8_t*>(&d);
+    out_.writeBytes(p, sizeof(double));
+}
+
+void BinaryEncoder::encodeString(const std::string& s)
+{
+    doEncodeLong(s.size());
+    out_.writeBytes(reinterpret_cast<const uint8_t*>(s.c_str()), s.size());
+}
+
+void BinaryEncoder::encodeBytes(const uint8_t *bytes, size_t len)
+{
+    doEncodeLong(len);
+    out_.writeBytes(bytes, len);
+}
+
+void BinaryEncoder::encodeFixed(const uint8_t *bytes, size_t len)
+{
+    out_.writeBytes(bytes, len);
+}
+
+void BinaryEncoder::encodeEnum(size_t e)
+{
+    doEncodeLong(e);
+}
+
+void BinaryEncoder::arrayStart()
+{
+}
+
+void BinaryEncoder::arrayEnd()
+{
+    doEncodeLong(0);
+}
+
+void BinaryEncoder::mapStart()
+{
+}
+
+void BinaryEncoder::mapEnd()
+{
+    doEncodeLong(0);
+}
+
+void BinaryEncoder::setItemCount(size_t count)
+{
+    if (count == 0) {
+        throw Exception("Count cannot be zero");
+    }
+    doEncodeLong(count);
+}
+
+void BinaryEncoder::startItem()
+{
+}
+
+void BinaryEncoder::encodeUnionIndex(size_t e)
+{
+    doEncodeLong(e);
+}
+
+void BinaryEncoder::doEncodeLong(int64_t l)
+{
+    boost::array<uint8_t, 10> bytes;
+    size_t size = encodeInt64(l, bytes);
+    out_.writeBytes(bytes.data(), size);
+}
+}   // namespace avro

Added: avro/trunk/lang/c++/impl/FileStream.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/FileStream.cc?rev=1072677&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/FileStream.cc (added)
+++ avro/trunk/lang/c++/impl/FileStream.cc Sun Feb 20 18:23:18 2011
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Stream.hh"
+#include "unistd.h"
+#include "fcntl.h"
+#include "errno.h"
+
+#ifndef O_BINARY
+#define O_BINARY 0
+#endif
+
+namespace avro {
+class FileInputStream : public InputStream {
+    const size_t bufferSize_;
+    uint8_t* const buffer_;
+    /// Input file descriptor
+    int in_;
+    size_t byteCount_;
+    uint8_t* next_;
+    size_t available_;
+
+    bool next(const uint8_t** data, size_t *size) {
+        if (available_ == 0 && ! fill()) {
+            return false;
+        }
+        *data = next_;
+        *size = available_;
+        next_ += available_;
+        byteCount_ += available_;
+        available_ = 0;
+        return true;
+    }
+
+    void backup(size_t len) {
+        next_ -= len;
+        available_ += len;
+        byteCount_ -= len;
+    }
+
+    void skip(size_t len) {
+        while (len > 0) {
+            if (available_ == 0) {
+                off_t r = ::lseek(in_, len, SEEK_CUR);
+                if (r == static_cast<off_t>(-1)) {
+                    throw Exception(boost::format("Cannot skip file: %1%") %
+                        strerror(errno));
+                }
+                byteCount_ += len;
+                return;
+            }
+            size_t n = std::min(available_, len);
+            available_ -= n;
+            next_ += n;
+            len -= n;
+            byteCount_ += n;
+        }
+    }
+
+    size_t byteCount() const { return byteCount_; }
+
+    bool fill() {
+        int n = ::read(in_, buffer_, bufferSize_);
+        if (n > 0) {
+            next_ = buffer_;
+            available_ = n;
+            return true;
+        }
+        return false;
+    }
+
+
+public:
+    FileInputStream(const char* filename, size_t bufferSize) :
+        bufferSize_(bufferSize),
+        buffer_(new uint8_t[bufferSize]),
+        in_(open(filename, O_RDONLY | O_BINARY)),
+        byteCount_(0),
+        next_(buffer_),
+        available_(0) {
+        if (in_ < 0) {
+            throw Exception(boost::format("Cannot open file: %1%") %
+                ::strerror(errno));
+        }
+    }
+
+    ~FileInputStream() {
+        ::close(in_);
+        delete[] buffer_;
+    }
+};
+
+class FileOutputStream : public OutputStream {
+    size_t bufferSize_;
+    uint8_t* const buffer_;
+    int out_;
+    uint8_t* next_;
+    size_t available_;
+    size_t byteCount_;
+
+    bool next(uint8_t** data, size_t* len) {
+        if (available_ == 0) {
+            flush();
+        }
+        *data = next_;
+        *len = available_;
+        byteCount_ += available_;
+        next_ += available_;
+        byteCount_ += available_;
+        available_ = 0;
+    }
+
+    void backup(size_t len) {
+        available_ += len;
+        next_ -= len;
+        byteCount_ -= len;
+    }
+
+    uint64_t byteCount() const {
+        return byteCount_;
+    }
+
+    void flush() {
+        if (::write(out_, buffer_, bufferSize_ - available_) < 0) {
+            throw Exception(boost::format("Cannot write file: %1%") %
+                strerror(errno));
+        }
+        next_ = buffer_;
+        available_ = bufferSize_;
+    }
+
+public:
+    FileOutputStream(const char* filename, size_t bufferSize) :
+        bufferSize_(bufferSize),
+        buffer_(new uint8_t[bufferSize]),
+        out_(::open(filename, O_WRONLY | O_CREAT | O_BINARY, 0644)),
+        next_(buffer_),
+        available_(bufferSize_) { }
+
+    ~FileOutputStream() {
+        if (out_ >= 0) {
+            ::close(out_);
+        }
+        delete[] buffer_;
+    }
+};
+
+std::auto_ptr<InputStream> fileInputStream(const char* filename,
+    size_t bufferSize)
+{
+    return std::auto_ptr<InputStream>(
+        new FileInputStream(filename, bufferSize));
+}
+
+std::auto_ptr<OutputStream> fileOutputStream(const char* filename,
+    size_t bufferSize)
+{
+    return std::auto_ptr<OutputStream>(
+        new FileOutputStream(filename, bufferSize));
+}
+
+}   // namespace avro

Added: avro/trunk/lang/c++/impl/Stream.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/Stream.cc?rev=1072677&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/Stream.cc (added)
+++ avro/trunk/lang/c++/impl/Stream.cc Sun Feb 20 18:23:18 2011
@@ -0,0 +1,180 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "Stream.hh"
+#include <vector>
+
+namespace avro {
+
+class MemoryInputStream : public InputStream {
+    const std::vector<uint8_t*>& data_;
+    const size_t chunkSize_;
+    const size_t size_;
+    const size_t available_;
+    size_t cur_;
+    size_t curLen_;
+
+    size_t maxLen() {
+        size_t n = (cur_ == (size_ - 1)) ? available_ : chunkSize_;
+        if (n == curLen_) {
+            if (cur_ == (size_ - 1)) {
+                return 0;
+            }
+            ++cur_;
+            n = (cur_ == (size_ - 1)) ? available_ : chunkSize_;
+            curLen_ = 0;
+        }
+        return n;
+    }
+
+public:
+    MemoryInputStream(const std::vector<uint8_t*>& b,
+        size_t chunkSize, size_t available) :
+            data_(b), chunkSize_(chunkSize), size_(b.size()),
+            available_(available), cur_(0), curLen_(0) { }
+
+    bool next(const uint8_t** data, size_t* len) {
+        if (size_t n = maxLen()) {
+            *data = data_[cur_] + curLen_;
+            *len = n - curLen_;
+            curLen_ = n;
+            return true;
+        }
+        return false;
+    }
+
+    void backup(size_t len) {
+        curLen_ -= len;
+    }
+
+    void skip(size_t len) {
+        while (len > 0) {
+            if (size_t n = maxLen()) {
+                if ((curLen_ + len) < n) {
+                    n = curLen_ + len;
+                }
+                len -= n - curLen_;
+                curLen_ = n;
+            } else {
+                break;
+            }
+        }
+    }
+
+    size_t byteCount() const {
+        return cur_ * chunkSize_ + curLen_;
+    }
+};
+
+class MemoryInputStream2 : public InputStream {
+    const uint8_t* const data_;
+    const size_t size_;
+    size_t curLen_;
+public:
+    MemoryInputStream2(const uint8_t *data, size_t len)
+        : data_(data), size_(len), curLen_(0) { }
+
+    bool next(const uint8_t** data, size_t* len) {
+        if (curLen_ == size_) {
+            return false;
+        }
+        *data = &data_[curLen_];
+        *len = size_ - curLen_;
+        curLen_ = size_;
+        return true;
+    }
+
+    void backup(size_t len) {
+        curLen_ -= len;
+    }
+
+    void skip(size_t len) {
+        if (len > (size_ - curLen_)) {
+            len = size_ - curLen_;
+        }
+        curLen_ += len;
+    }
+
+    size_t byteCount() const {
+        return curLen_;
+    }
+};
+
+class MemoryOutputStream : public OutputStream {
+public:
+    const size_t chunkSize_;
+    std::vector<uint8_t*> data_;
+    size_t available_;
+    size_t byteCount_;
+
+    MemoryOutputStream(size_t chunkSize) : chunkSize_(chunkSize),
+        available_(0), byteCount_(0) { }
+    ~MemoryOutputStream() {
+        for (std::vector<uint8_t*>::const_iterator it = data_.begin();
+            it != data_.end(); ++it) {
+            delete[] *it;
+        }
+    }
+
+    bool next(uint8_t** data, size_t* len) {
+        if (available_ == 0) {
+            data_.push_back(new uint8_t[chunkSize_]);
+            available_ = chunkSize_;
+        }
+        *data = &data_.back()[chunkSize_ - available_];
+        *len = available_;
+        byteCount_ += available_;
+        available_ = 0;
+        return true;
+    }
+
+    void backup(size_t len) {
+        available_ += len;
+        byteCount_ -= len;
+    }
+
+    uint64_t byteCount() const {
+        return byteCount_;
+    }
+
+    void flush() { }
+};
+
+std::auto_ptr<OutputStream> memoryOutputStream(size_t chunkSize)
+{
+    return std::auto_ptr<OutputStream>(new MemoryOutputStream(chunkSize));
+}
+
+std::auto_ptr<InputStream> memoryInputStream(const uint8_t* data, size_t len)
+{
+    return std::auto_ptr<InputStream>(new MemoryInputStream2(data, len));
+}
+
+std::auto_ptr<InputStream> memoryInputStream(const OutputStream& source)
+{
+    const MemoryOutputStream& mos =
+        dynamic_cast<const MemoryOutputStream&>(source);
+    return (mos.data_.empty()) ?
+        std::auto_ptr<InputStream>(new MemoryInputStream2(0, 0)) :
+        std::auto_ptr<InputStream>(new MemoryInputStream(mos.data_,
+            mos.chunkSize_,
+            (mos.chunkSize_ - mos.available_)));
+}
+
+}   // namespace avro
+

Modified: avro/trunk/lang/c++/impl/Types.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/Types.cc?rev=1072677&r1=1072676&r2=1072677&view=diff
==============================================================================
--- avro/trunk/lang/c++/impl/Types.cc (original)
+++ avro/trunk/lang/c++/impl/Types.cc Sun Feb 20 18:23:18 2011
@@ -50,6 +50,16 @@ BOOST_STATIC_ASSERT( (sizeof(typeToStrin
 // and it would be a problem for this flag if we ever supported more than 32 types
 BOOST_STATIC_ASSERT( AVRO_NUM_TYPES < 32 );
 
+const std::string& toString(Type type)
+{
+    static std::string undefinedType = "Undefined type";
+    if (isAvroTypeOrPseudoType(type)) {
+        return strings::typeToString[type];
+    } else {
+        return undefinedType;
+    }
+}
+
 std::ostream &operator<< (std::ostream &os, Type type)
 {
     if(isAvroTypeOrPseudoType(type)) {

Added: avro/trunk/lang/c++/impl/parsing/JsonCodec.cc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/c%2B%2B/impl/parsing/JsonCodec.cc?rev=1072677&view=auto
==============================================================================
--- avro/trunk/lang/c++/impl/parsing/JsonCodec.cc (added)
+++ avro/trunk/lang/c++/impl/parsing/JsonCodec.cc Sun Feb 20 18:23:18 2011
@@ -0,0 +1,1269 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define __STDC_LIMIT_MACROS
+
+#include <string>
+#include <stack>
+#include <map>
+#include <algorithm>
+#include <ctype.h>
+#include <boost/shared_ptr.hpp>
+#include <boost/make_shared.hpp>
+#include <boost/weak_ptr.hpp>
+#include <boost/any.hpp>
+#include <boost/utility.hpp>
+
+#include "ValidatingCodec.hh"
+#include "Symbol.hh"
+#include "ValidSchema.hh"
+#include "Decoder.hh"
+#include "Encoder.hh"
+#include "NodeImpl.hh"
+
+namespace avro {
+
+using boost::make_shared;
+
+namespace parsing {
+
+using boost::shared_ptr;
+using boost::static_pointer_cast;
+
+using std::map;
+using std::vector;
+using std::string;
+using std::reverse;
+using std::ostringstream;
+using std::istringstream;
+using std::stack;
+
+class JsonGrammarGenerator : public ValidatingGrammarGenerator {
+    Production doGenerate(const NodePtr& n,
+        std::map<NodePtr, boost::shared_ptr<Production> > &m);
+};
+
+static std::string nameOf(const NodePtr& n)
+{
+    if (n->hasName()) {
+        return n->name();
+    }
+    std::ostringstream oss;
+    oss << n->type();
+    return oss.str();
+}
+
+Production JsonGrammarGenerator::doGenerate(const NodePtr& n,
+    std::map<NodePtr, boost::shared_ptr<Production> > &m) {
+    switch (n->type()) {
+    case AVRO_NULL:
+    case AVRO_BOOL:
+    case AVRO_INT:
+    case AVRO_LONG:
+    case AVRO_FLOAT:
+    case AVRO_DOUBLE:
+    case AVRO_STRING:
+    case AVRO_BYTES:
+    case AVRO_FIXED:
+    case AVRO_ARRAY:
+    case AVRO_MAP:
+    case AVRO_SYMBOLIC:
+        return ValidatingGrammarGenerator::doGenerate(n, m);
+    case AVRO_RECORD:
+        {
+            Production result;
+
+            m.erase(n);
+
+            size_t c = n->leaves();
+            result.reserve(2 + 2 * c);
+            result.push_back(Symbol::recordStartSymbol());
+            for (size_t i = 0; i < c; ++i) {
+                const NodePtr& leaf = n->leafAt(i);
+                Production v = doGenerate(leaf, m);
+                result.push_back(Symbol::fieldSymbol(n->nameAt(i)));
+                copy(v.rbegin(), v.rend(), back_inserter(result));
+            }
+            result.push_back(Symbol::recordEndSymbol());
+            reverse(result.begin(), result.end());
+
+            bool found = m.find(n) != m.end();
+
+            shared_ptr<Production> p = make_shared<Production>(result);
+            m[n] = p;
+
+            return found ? Production(1, Symbol::indirect(p)) : result;
+        }
+    case AVRO_ENUM:
+        {
+            vector<string> nn;
+            size_t c = n->names();
+            nn.reserve(c);
+            for (int i = 0; i < c; ++i) {
+                nn.push_back(n->nameAt(i));
+            }
+            Symbol r[] = {
+                Symbol::nameListSymbol(nn),
+                Symbol::enumSymbol() };
+            Production result(r, r + 2);
+            m[n] = boost::make_shared<Production>(result);
+            return result;
+        }
+    case AVRO_UNION:
+        {
+            size_t c = n->leaves();
+
+            vector<Production> vv;
+            vv.reserve(c);
+
+            vector<string> names;
+            names.reserve(c);
+
+            for (size_t i = 0; i < c; ++i) {
+                const NodePtr& nn = n->leafAt(i);
+                Production v = doGenerate(nn, m);
+                if (nn->type() != AVRO_NULL) {
+                    Production v2;
+                    v2.push_back(Symbol::recordEndSymbol());
+                    copy(v.begin(), v.end(), back_inserter(v2));
+                    v.swap(v2);
+                }
+                vv.push_back(v);
+                names.push_back(nameOf(nn));
+            }
+            Symbol r[] = {
+                Symbol::alternative(vv),
+                Symbol::nameListSymbol(names),
+                Symbol::unionSymbol()
+            };
+            return Production(r, r + 3);
+        }
+    default:
+        throw Exception("Unknown node type");
+    }
+}
+
+static char toHex(unsigned int n) {
+    return (n < 10) ? (n + '0') : (n + 'a' - 10);
+}
+
+class JsonParser : boost::noncopyable {
+public:
+    enum Token {
+        tkNull,
+        tkBool,
+        tkLong,
+        tkDouble,
+        tkString,
+        tkArrayStart,
+        tkArrayEnd,
+        tkObjectStart,
+        tkObjectEnd
+    };
+
+private:
+    enum State {
+        stValue,    // Expect a data type
+        stArray0,   // Expect a data type or ']'
+        stArrayN,   // Expect a ',' or ']'
+        stObject0,  // Expect a string or a '}'
+        stObjectN,  // Expect a ',' or '}'
+        stKey       // Expect a ':'
+    };
+    stack<State> stateStack;
+    State curState;
+    bool hasNext;
+    char nextChar;
+    bool peeked;
+
+    StreamReader in_;
+    Token curToken;
+    bool bv;
+    int64_t lv;
+    double dv;
+    string sv;
+
+    Token doAdvance();
+    Token tryLiteral(const char exp[], size_t n, Token tk);
+    Token tryNumber(char ch);
+    Token tryString();
+    void unexpected(unsigned char ch);
+    char next();
+
+public:
+    JsonParser() : curState(stValue), hasNext(false), peeked(false) { }
+
+    void init(InputStream& is) {
+        in_.reset(is);
+    }
+
+    Token advance() {
+        if (! peeked) {
+            curToken = doAdvance();
+        } else {
+            peeked = false;
+        }
+        return curToken;
+    }
+
+    Token peek() {
+        if (! peeked) {
+            curToken = doAdvance();
+            peeked = true;
+        }
+        return curToken;
+    }
+
+    bool boolValue() {
+        return bv;
+    }
+
+    Token cur() {
+        return curToken;
+    }
+
+    double doubleValue() {
+        return dv;
+    }
+
+    int64_t longValue() {
+        return lv;
+    }
+
+    string stringValue() {
+        return sv;
+    }
+
+    static const char* const tokenNames[];
+
+    static const char* toString(Token tk) {
+        return tokenNames[tk];
+    }
+};
+
+const char* const
+JsonParser::tokenNames[] = {
+    "Null",
+    "Bool",
+    "Integer",
+    "Double",
+    "String",
+    "Array start",
+    "Array end",
+    "Object start",
+    "Object end",
+};
+
+char JsonParser::next()
+{
+    char ch = hasNext ? nextChar : ' ';
+    while (isspace(ch)) {
+        ch = in_.read();
+    }
+    hasNext = false;
+    return ch;
+}
+
+JsonParser::Token JsonParser::doAdvance()
+{
+    char ch = next();
+    if (ch == ']') {
+        if (curState == stArray0 || stArrayN) {
+            curState = stateStack.top();
+            stateStack.pop();
+            return tkArrayEnd;
+        } else {
+            unexpected(ch);
+        }
+    } else if (ch == '}') {
+        if (curState == stObject0 || stObjectN) {
+            curState = stateStack.top();
+            stateStack.pop();
+            return tkObjectEnd;
+        } else {
+            unexpected(ch);
+        }
+    } else if (ch == ',') {
+        if (curState != stObjectN && curState != stArrayN) {
+            unexpected(ch);
+        }
+        if (curState == stObjectN) {
+            curState = stObject0;
+        }
+        ch = next();
+    } else if (ch == ':') {
+        if (curState != stKey) {
+            unexpected(ch);
+        }
+        curState = stObjectN;
+        ch = next();
+    }
+
+    if (curState == stObject0) {
+        if (ch != '"') {
+            unexpected(ch);
+        }
+        curState = stKey;
+    } else if (curState == stArray0) {
+        curState = stArrayN;
+    }
+
+    switch (ch) {
+    case '[':
+        stateStack.push(curState);
+        curState = stArray0;
+        return tkArrayStart;
+    case '{':
+        stateStack.push(curState);
+        curState = stObject0;
+        return tkObjectStart;
+    case '"':
+        return tryString();
+    case 't':
+        bv = true;
+        return tryLiteral("rue", 3, tkBool);
+    case 'f':
+        bv = false;
+        return tryLiteral("alse", 4, tkBool);
+    case 'n':
+        return tryLiteral("ull", 3, tkNull);
+    default:
+        if (isdigit(ch) || '-') {
+            return tryNumber(ch);
+        } else {
+            unexpected(ch);
+        }
+    }
+}
+
+JsonParser::Token JsonParser::tryNumber(char ch)
+{
+    sv.clear();
+    sv.push_back(ch);
+
+    hasNext = false;
+    int state = (ch == '-') ? 0 : (ch == 0) ? 1 : 2;
+    for (; ;) {
+        switch (state) {
+        case 0:
+            if (in_.hasMore()) {
+                ch = in_.read();
+                if (isdigit(ch)) {
+                    state = (ch == 0) ? 1 : 2;
+                    sv.push_back(ch);
+                    continue;
+                }
+                hasNext = true;
+            }
+            break;
+        case 1:
+            if (in_.hasMore()) {
+                ch = in_.read();
+                if (ch == '.') {
+                    state = 3;
+                    sv.push_back(ch);
+                    continue;
+                }
+                hasNext = true;
+            }
+            break;
+        case 2:
+            if (in_.hasMore()) {
+                ch = in_.read();
+                if (isdigit(ch)) {
+                    sv.push_back(ch);
+                    continue;
+                } else if (ch == '.') {
+                    state = 3;
+                    sv.push_back(ch);
+                    continue;
+                }
+                hasNext = true;
+            }
+            break;
+        case 3:
+        case 6:
+            if (in_.hasMore()) {
+                ch = in_.read();
+                if (isdigit(ch)) {
+                    sv.push_back(ch);
+                    state++;
+                    continue;
+                }
+                hasNext = true;
+            }
+            break;
+        case 4:
+            if (in_.hasMore()) {
+                ch = in_.read();
+                if (isdigit(ch)) {
+                    sv.push_back(ch);
+                    continue;
+                } else if (ch == 'e' || ch == 'E') {
+                    sv.push_back(ch);
+                    state = 5;
+                    continue;
+                }
+                hasNext = true;
+            }
+            break;
+        case 5:
+            if (in_.hasMore()) {
+                in_.read();
+                ch = next();
+                if (ch == '+' || ch == '-') {
+                    sv.push_back(ch);
+                    state = 6;
+                    continue;
+                } else if (isdigit(ch)) {
+                    sv.push_back(ch);
+                    state = 7;
+                    continue;
+                }
+                hasNext = true;
+            }
+            break;
+        case 7:
+            if (in_.hasMore()) {
+                in_.read();
+                ch = next();
+                if (isdigit(ch)) {
+                    sv.push_back(ch);
+                    continue;
+                }
+                hasNext = true;
+            }
+            break;
+        }
+        if (state == 1 || state == 2 || state == 4 || state == 7) {
+            if (hasNext) {
+                nextChar = ch;
+            }
+            istringstream iss(sv);
+            if (state == 1 || state == 2) {
+                iss >> lv;
+                return tkLong;
+            } else {
+                iss >> dv;
+                return tkDouble;
+            }
+        } else {
+            if (hasNext) {
+                unexpected(ch);
+            } else {
+                throw Exception("Unexpected EOF");
+            }
+        }
+    }
+}
+
+JsonParser::Token JsonParser::tryString()
+{
+    sv.clear();
+    for ( ; ;) {
+        char ch = in_.read();
+        if (ch == '"') {
+            return tkString;
+        } else if (ch == '\\') {
+            ch = in_.read();
+            switch (ch) {
+            case '"':
+            case '\\':
+            case '/':
+                sv.push_back(ch);
+                continue;
+            case 'b':
+                sv.push_back('\b');
+                continue;
+            case 'f':
+                sv.push_back('\f');
+                continue;
+            case 'n':
+                sv.push_back('\n');
+                continue;
+            case 'r':
+                sv.push_back('\r');
+                continue;
+            case 't':
+                sv.push_back('\t');
+                continue;
+            case 'U':
+                {
+                    unsigned int n = 0;
+                    char e[4];
+                    in_.readBytes(reinterpret_cast<uint8_t*>(e), 4);
+                    for (int i = 0; i < 4; i++) {
+                        n *= 16;
+                        char c = e[i];
+                        if (isdigit(c)) {
+                            n += c - '0';
+                        } else if (c >= 'a' && c <= 'f') {
+                            n += c - 'a' + 10;
+                        } else if (c >= 'A' && c <= 'F') {
+                            n += c - 'A' + 10;
+                        } else {
+                            unexpected(c);
+                        }
+                    }
+                    sv.push_back(n);
+                }
+                break;
+            default:
+                unexpected(ch);
+            }
+        } else {
+            sv.push_back(ch);
+        }
+    }
+}
+
+void JsonParser::unexpected(unsigned char c)
+{
+    ostringstream oss;
+    oss << "Unexpected character in json " << toHex(c / 16) << toHex(c % 16);
+    throw Exception(oss.str());
+}
+
+JsonParser::Token JsonParser::tryLiteral(const char exp[], size_t n, Token tk)
+{
+    char c[100];
+    in_.readBytes(reinterpret_cast<uint8_t*>(c), n);
+    for (int i = 0; i < n; ++i) {
+        if (c[i] != exp[i]) {
+            unexpected(c[i]);
+        }
+    }
+    if (in_.hasMore()) {
+        nextChar = in_.read();
+        if (isdigit(nextChar) || isalpha(nextChar)) {
+            unexpected(nextChar);
+        }
+        hasNext = true;
+    }
+    return tk;
+}
+
+static void expectToken(JsonParser& in, JsonParser::Token tk)
+{
+    if (in.advance() != tk) {
+        ostringstream oss;
+        oss << "Incorrect token in the stream. Expected: "
+            << JsonParser::toString(tk) << ", found "
+            << JsonParser::toString(in.cur());
+        throw Exception(oss.str());
+    }
+}
+
+class JsonDecoderHandler {
+    JsonParser& in_;
+public:
+    JsonDecoderHandler(JsonParser& p) : in_(p) { }
+    size_t handle(const Symbol& s) {
+        switch (s.kind()) {
+        case Symbol::sRecordStart:
+            expectToken(in_, JsonParser::tkObjectStart);
+            break;
+        case Symbol::sRecordEnd:
+            expectToken(in_, JsonParser::tkObjectEnd);
+            break;
+        case Symbol::sField:
+            expectToken(in_, JsonParser::tkString);
+            if (s.extra<string>() != in_.stringValue()) {
+                throw Exception("Incorrect field");
+            }
+            break;
+        }
+        return 0;
+    }
+};
+
+template <typename P>
+class JsonDecoder : public Decoder {
+    JsonParser in_;
+    JsonDecoderHandler handler_;
+    P parser_;
+
+    void init(InputStream& is);
+    void decodeNull();
+    bool decodeBool();
+    int32_t decodeInt();
+    int64_t decodeLong();
+    float decodeFloat();
+    double decodeDouble();
+    void decodeString(string& value);
+    void skipString();
+    void decodeBytes(vector<uint8_t>& value);
+    void skipBytes();
+    void decodeFixed(size_t n, vector<uint8_t>& value);
+    void skipFixed(size_t n);
+    size_t decodeEnum();
+    size_t arrayStart();
+    size_t arrayNext();
+    size_t skipArray();
+    size_t mapStart();
+    size_t mapNext();
+    size_t skipMap();
+    size_t decodeUnionIndex();
+
+    void expect(JsonParser::Token tk);
+    void skipComposite();
+public:
+
+    JsonDecoder(const ValidSchema& s) :
+        handler_(in_),
+        parser_(JsonGrammarGenerator().generate(s), NULL, handler_) { }
+
+};
+
+template <typename P>
+void JsonDecoder<P>::init(InputStream& is)
+{
+    in_.init(is);
+}
+
+template <typename P>
+void JsonDecoder<P>::expect(JsonParser::Token tk)
+{
+    expectToken(in_, tk);
+}
+
+template <typename P>
+void JsonDecoder<P>::decodeNull()
+{
+    parser_.advance(Symbol::sNull);
+    expect(JsonParser::tkNull);
+}
+
+template <typename P>
+bool JsonDecoder<P>::decodeBool()
+{
+    parser_.advance(Symbol::sBool);
+    expect(JsonParser::tkBool);
+    bool result = in_.boolValue();
+    return result;
+}
+
+template <typename P>
+int32_t JsonDecoder<P>::decodeInt()
+{
+    parser_.advance(Symbol::sInt);
+    expect(JsonParser::tkLong);
+    int64_t result = in_.longValue();
+    if (result < INT32_MIN || result > INT32_MAX) {
+        throw Exception(boost::format("Value out of range for Avro int: %1%")
+            % result);
+    }
+    return static_cast<int32_t>(result);
+}
+
+template <typename P>
+int64_t JsonDecoder<P>::decodeLong()
+{
+    parser_.advance(Symbol::sLong);
+    expect(JsonParser::tkLong);
+    int64_t result = in_.longValue();
+    return result;
+}
+
+template <typename P>
+float JsonDecoder<P>::decodeFloat()
+{
+    parser_.advance(Symbol::sFloat);
+    expect(JsonParser::tkDouble);
+    double result = in_.doubleValue();
+    return static_cast<float>(result);
+}
+
+template <typename P>
+double JsonDecoder<P>::decodeDouble()
+{
+    parser_.advance(Symbol::sDouble);
+    expect(JsonParser::tkDouble);
+    double result = in_.doubleValue();
+    return result;
+}
+
+template <typename P>
+void JsonDecoder<P>::decodeString(string& value)
+{
+    parser_.advance(Symbol::sString);
+    expect(JsonParser::tkString);
+    value = in_.stringValue();
+}
+
+template <typename P>
+void JsonDecoder<P>::skipString()
+{
+    parser_.advance(Symbol::sString);
+    expect(JsonParser::tkString);
+}
+
+static vector<uint8_t> toBytes(const string& s)
+{
+    return vector<uint8_t>(s.begin(), s.end());
+}
+
+template <typename P>
+void JsonDecoder<P>::decodeBytes(vector<uint8_t>& value )
+{
+    parser_.advance(Symbol::sBytes);
+    expect(JsonParser::tkString);
+    value = toBytes(in_.stringValue());
+}
+
+template <typename P>
+void JsonDecoder<P>::skipBytes()
+{
+    parser_.advance(Symbol::sBytes);
+    expect(JsonParser::tkString);
+}
+
+template <typename P>
+void JsonDecoder<P>::decodeFixed(size_t n, vector<uint8_t>& value)
+{
+    parser_.advance(Symbol::sFixed);
+    parser_.assertSize(n);
+    expect(JsonParser::tkString);
+    value = toBytes(in_.stringValue());
+    if (value.size() != n) {
+        throw Exception("Incorrect value for fixed");
+    }
+}
+
+template <typename P>
+void JsonDecoder<P>::skipFixed(size_t n)
+{
+    parser_.advance(Symbol::sFixed);
+    parser_.assertSize(n);
+    expect(JsonParser::tkString);
+    vector<uint8_t> result = toBytes(in_.stringValue());
+    if (result.size() != n) {
+        throw Exception("Incorrect value for fixed");
+    }
+}
+
+template <typename P>
+size_t JsonDecoder<P>::decodeEnum()
+{
+    parser_.advance(Symbol::sEnum);
+    expect(JsonParser::tkString);
+    size_t result = parser_.indexForName(in_.stringValue());
+    return result;
+}
+
+template <typename P>
+size_t JsonDecoder<P>::arrayStart()
+{
+    parser_.advance(Symbol::sArrayStart);
+    expect(JsonParser::tkArrayStart);
+    return arrayNext();
+}
+
+template <typename P>
+size_t JsonDecoder<P>::arrayNext()
+{
+    parser_.processImplicitActions();
+    if (in_.peek() == JsonParser::tkArrayEnd) {
+        in_.advance();
+        parser_.popRepeater();
+        parser_.advance(Symbol::sArrayEnd);
+        return 0;
+    }
+    parser_.setRepeatCount(1);
+    return 1;
+}
+
+template<typename P>
+void JsonDecoder<P>::skipComposite()
+{
+    size_t level = 0;
+    for (; ;) {
+        switch (in_.advance()) {
+        case JsonParser::tkArrayStart:
+        case JsonParser::tkObjectStart:
+            ++level;
+            continue;
+        case JsonParser::tkArrayEnd:
+        case JsonParser::tkObjectEnd:
+            if (level == 0) {
+                return;
+            }
+            --level;
+            continue;
+        default:
+            continue;
+        }
+    }
+}
+
+template <typename P>
+size_t JsonDecoder<P>::skipArray()
+{
+    parser_.advance(Symbol::sArrayStart);
+    parser_.pop();
+    parser_.advance(Symbol::sArrayEnd);
+    expect(JsonParser::tkArrayStart);
+    skipComposite();
+    return 0;
+}
+
+template <typename P>
+size_t JsonDecoder<P>::mapStart()
+{
+    parser_.advance(Symbol::sMapStart);
+    expect(JsonParser::tkObjectStart);
+    return mapNext();
+}
+
+template <typename P>
+size_t JsonDecoder<P>::mapNext()
+{
+    parser_.processImplicitActions();
+    if (in_.peek() == JsonParser::tkObjectEnd) {
+        in_.advance();
+        parser_.popRepeater();
+        parser_.advance(Symbol::sMapEnd);
+        return 0;
+    }
+    parser_.setRepeatCount(1);
+    return 1;
+}
+
+template <typename P>
+size_t JsonDecoder<P>::skipMap()
+{
+    parser_.advance(Symbol::sMapStart);
+    parser_.pop();
+    parser_.advance(Symbol::sMapEnd);
+    expect(JsonParser::tkObjectStart);
+    skipComposite();
+    return 0;
+}
+
+template <typename P>
+size_t JsonDecoder<P>::decodeUnionIndex()
+{
+    parser_.advance(Symbol::sUnion);
+
+    size_t result;
+    if (in_.peek() == JsonParser::tkNull) {
+        result = parser_.indexForName("null");
+    } else {
+        expect(JsonParser::tkObjectStart);
+        expect(JsonParser::tkString);
+        result = parser_.indexForName(in_.stringValue());
+    }
+    parser_.selectBranch(result);
+    return result;
+}
+
+class JsonGenerator {
+    StreamWriter out_;
+    enum State {
+        stStart,
+        stArray0,
+        stArrayN,
+        stMap0,
+        stMapN,
+        stKey,
+    };
+
+    stack<State> stateStack;
+    State top;
+
+    void write(const char *b, const char* p) {
+        if (b != p) {
+            out_.writeBytes(reinterpret_cast<const uint8_t*>(b), p - b);
+        }
+    }
+
+    void escape(char c, const char* b, const char *p) {
+        write(b, p);
+        out_.write('\\');
+        out_.write(c);
+    }
+
+    void escapeCtl(char c) {
+        out_.write('\\');
+        out_.write('U');
+        out_.write('0');
+        out_.write('0');
+        out_.write(toHex((static_cast<unsigned char>(c)) / 16));
+        out_.write(toHex((static_cast<unsigned char>(c)) % 16));
+    }
+
+    void doEncodeString(const std::string& s) {
+        const char* b = &s[0];
+        const char* e = b + s.size();
+        out_.write('"');
+        for (const char* p = b; p != e; p++) {
+            switch (*p) {
+            case '\\':
+            case '"':
+            case '/':
+                escape(*p, b, p);
+                break;
+            case '\b':
+                escape('b', b, p);
+                break;
+            case '\f':
+                escape('f', b, p);
+                break;
+            case '\n':
+                escape('n', b, p);
+                break;
+            case '\r':
+                escape('r', b, p);
+                break;
+            case '\t':
+                escape('t', b, p);
+                break;
+            default:
+                if (! iscntrl(*p)) {
+                    continue;
+                }
+                write(b, p);
+                escapeCtl(*p);
+                break;
+            }
+            b = p + 1;
+        }
+        write(b, e);
+        out_.write('"');
+    }
+
+    void sep() {
+        if (top == stArrayN) {
+            out_.write(',');
+        } else if (top == stArray0) {
+            top = stArrayN;
+        }
+    }
+
+    void sep2() {
+        if (top == stKey) {
+            top = stMapN;
+        }
+    }
+
+public:
+    JsonGenerator() : top(stStart) { }
+
+    void init(OutputStream& os) {
+        out_.reset(os);
+    }
+
+    void flush() {
+        out_.flush();
+    }
+
+    void encodeNull() {
+        sep();
+        out_.writeBytes(reinterpret_cast<const uint8_t*>("null"), 4);
+        sep2();
+    }
+
+    void encodeBool(bool b) {
+        sep();
+        if (b) {
+            out_.writeBytes(reinterpret_cast<const uint8_t*>("true"), 4);
+        } else {
+            out_.writeBytes(reinterpret_cast<const uint8_t*>("false"), 5);
+        }
+        sep2();
+    }
+
+    template <typename T>
+    void encodeNumber(T t) {
+        sep();
+        ostringstream oss;
+        oss << t;
+        const std::string& s = oss.str();
+        out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size());
+        sep2();
+    }
+
+    void encodeString(const std::string& s) {
+        if (top == stMap0) {
+            top = stKey;
+        } else if (top == stMapN) {
+            out_.write(',');
+            top = stKey;
+        } else if (top == stKey) {
+            top = stMapN;
+        } else {
+            sep();
+        }
+        doEncodeString(s);
+        if (top == stKey) {
+            out_.write(':');
+        }
+    }
+
+    void encodeBinary(const uint8_t* bytes, size_t len) {
+        sep();
+        out_.write('"');
+        const uint8_t* e = bytes + len;
+        while (bytes != e) {
+            escapeCtl(*bytes++);
+        }
+        out_.write('"');
+        sep2();
+    }
+
+    void arrayStart() {
+        sep();
+        stateStack.push(top);
+        top = stArray0;
+        out_.write('[');
+    }
+
+    void arrayEnd() {
+        top = stateStack.top();
+        stateStack.pop();
+        out_.write(']');
+        sep2();
+    }
+
+    void objectStart() {
+        sep();
+        stateStack.push(top);
+        top = stMap0;
+        out_.write('{');
+    }
+
+    void objectEnd() {
+        top = stateStack.top();
+        stateStack.pop();
+        out_.write('}');
+        sep2();
+    }
+
+};
+
+class JsonHandler {
+    JsonGenerator& generator_;
+public:
+    JsonHandler(JsonGenerator& g) : generator_(g) { }
+    size_t handle(const Symbol& s) {
+        switch (s.kind()) {
+        case Symbol::sRecordStart:
+            generator_.objectStart();
+            break;
+        case Symbol::sRecordEnd:
+            generator_.objectEnd();
+            break;
+        case Symbol::sField:
+            generator_.encodeString(s.extra<string>());
+            break;
+        }
+        return 0;
+    }
+};
+
+template <typename P>
+class JsonEncoder : public Encoder {
+    JsonGenerator out_;
+    JsonHandler handler_;
+    P parser_;
+
+    void init(OutputStream& os);
+    void flush();
+    void encodeNull();
+    void encodeBool(bool b);
+    void encodeInt(int32_t i);
+    void encodeLong(int64_t l);
+    void encodeFloat(float f);
+    void encodeDouble(double d);
+    void encodeString(const std::string& s);
+    void encodeBytes(const uint8_t *bytes, size_t len);
+    void encodeFixed(const uint8_t *bytes, size_t len);
+    void encodeEnum(size_t e);
+    void arrayStart();
+    void arrayEnd();
+    void mapStart();
+    void mapEnd();
+    void setItemCount(size_t count);
+    void startItem();
+    void encodeUnionIndex(size_t e);
+public:
+    JsonEncoder(const ValidSchema& schema) :
+        handler_(out_),
+        parser_(JsonGrammarGenerator().generate(schema), NULL, handler_) { }
+};
+
+template<typename P>
+void JsonEncoder<P>::init(OutputStream& os)
+{
+    out_.init(os);
+}
+
+template<typename P>
+void JsonEncoder<P>::flush()
+{
+    parser_.processImplicitActions();
+    out_.flush();
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeNull()
+{
+    parser_.advance(Symbol::sNull);
+    out_.encodeNull();
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeBool(bool b)
+{
+    parser_.advance(Symbol::sBool);
+    out_.encodeBool(b);
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeInt(int32_t i)
+{
+    parser_.advance(Symbol::sInt);
+    out_.encodeNumber(i);
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeLong(int64_t l)
+{
+    parser_.advance(Symbol::sLong);
+    out_.encodeNumber(l);
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeFloat(float f)
+{
+    parser_.advance(Symbol::sFloat);
+    out_.encodeNumber(f);
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeDouble(double d)
+{
+    parser_.advance(Symbol::sDouble);
+    out_.encodeNumber(d);
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeString(const std::string& s)
+{
+    parser_.advance(Symbol::sString);
+    out_.encodeString(s);
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeBytes(const uint8_t *bytes, size_t len)
+{
+    parser_.advance(Symbol::sBytes);
+    out_.encodeBinary(bytes, len);
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeFixed(const uint8_t *bytes, size_t len)
+{
+    parser_.advance(Symbol::sFixed);
+    parser_.assertSize(len);
+    out_.encodeBinary(bytes, len);
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeEnum(size_t e)
+{
+    parser_.advance(Symbol::sEnum);
+    const string& s = parser_.nameForIndex(e);
+    out_.encodeString(s);
+}
+
+template<typename P>
+void JsonEncoder<P>::arrayStart()
+{
+    parser_.advance(Symbol::sArrayStart);
+    out_.arrayStart();
+}
+
+template<typename P>
+void JsonEncoder<P>::arrayEnd()
+{
+    parser_.popRepeater();
+    parser_.advance(Symbol::sArrayEnd);
+    out_.arrayEnd();
+}
+
+template<typename P>
+void JsonEncoder<P>::mapStart()
+{
+    parser_.advance(Symbol::sMapStart);
+    out_.objectStart();
+}
+
+template<typename P>
+void JsonEncoder<P>::mapEnd()
+{
+    parser_.popRepeater();
+    parser_.advance(Symbol::sMapEnd);
+    out_.objectEnd();
+}
+
+template<typename P>
+void JsonEncoder<P>::setItemCount(size_t count)
+{
+    parser_.setRepeatCount(count);
+}
+
+template<typename P>
+void JsonEncoder<P>::startItem()
+{
+    parser_.processImplicitActions();
+    if (parser_.top() != Symbol::sRepeater) {
+        throw Exception("startItem at not an item boundary");
+    }
+}
+
+template<typename P>
+void JsonEncoder<P>::encodeUnionIndex(size_t e)
+{
+    parser_.advance(Symbol::sUnion);
+
+    const std::string name = parser_.nameForIndex(e);
+
+    if (name != "null") {
+        out_.objectStart();
+        out_.encodeString(name);
+    }
+    parser_.selectBranch(e);
+}
+
+}   // namespace parsing
+
+DecoderPtr jsonDecoder(const ValidSchema& s)
+{
+    return make_shared<parsing::JsonDecoder<
+        parsing::SimpleParser<parsing::JsonDecoderHandler> > >(s);
+}
+
+EncoderPtr jsonEncoder(const ValidSchema& schema)
+{
+    return make_shared<parsing::JsonEncoder<
+        parsing::SimpleParser<parsing::JsonHandler> > >(schema);
+}
+
+}   // namespace avro
+



Mime
View raw message