avro-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From th...@apache.org
Subject avro git commit: AVRO-1784. C++: Should have a json encoder that pretty prints
Date Wed, 13 Apr 2016 05:31:50 GMT
Repository: avro
Updated Branches:
  refs/heads/master 32067dec2 -> e98caa993


AVRO-1784. C++: Should have a json encoder that pretty prints


Project: http://git-wip-us.apache.org/repos/asf/avro/repo
Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/e98caa99
Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/e98caa99
Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/e98caa99

Branch: refs/heads/master
Commit: e98caa9937c47a4b44c80e0497995fd404102755
Parents: 32067de
Author: Thiruvalluvan M. G <thiru@verticloud.com>
Authored: Wed Apr 13 10:53:29 2016 +0530
Committer: Thiruvalluvan M. G <thiru@verticloud.com>
Committed: Wed Apr 13 10:58:22 2016 +0530

----------------------------------------------------------------------
 CHANGES.txt                        |  2 +
 lang/c++/api/Encoder.hh            |  7 ++-
 lang/c++/impl/json/JsonDom.cc      |  4 +-
 lang/c++/impl/json/JsonDom.hh      |  5 +-
 lang/c++/impl/json/JsonIO.cc       | 19 -------
 lang/c++/impl/json/JsonIO.hh       | 78 +++++++++++++++++++++++++-
 lang/c++/impl/parsing/JsonCodec.cc | 97 ++++++++++++++++++---------------
 lang/c++/test/CodecTests.cc        | 14 ++++-
 8 files changed, 154 insertions(+), 72 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b44745f..1399a94 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -89,6 +89,8 @@ Avro 1.8.0 (22 January 2016)
 
     AVRO-1767. JS: Fall back to unwrapped union values. (Matthieu Monsch via blue)
 
+    AVRO-1784. C++: Should have a json encoder that pretty prints. (John McClean via thiru)
+
   OPTIMIZATIONS
 
     AVRO-1760. Java: Thread scalability problem with the use of SynchronizedMap

http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/api/Encoder.hh
----------------------------------------------------------------------
diff --git a/lang/c++/api/Encoder.hh b/lang/c++/api/Encoder.hh
index a5dce71..e13fa69 100644
--- a/lang/c++/api/Encoder.hh
+++ b/lang/c++/api/Encoder.hh
@@ -156,10 +156,15 @@ AVRO_DECL EncoderPtr validatingEncoder(const ValidSchema& schema,
     const EncoderPtr& base);
 
 /**
- *  Returns an encoder that can encode Avro standard for JSON.
+ *  Returns an encoder that encodes Avro standard for JSON.
  */
 AVRO_DECL EncoderPtr jsonEncoder(const ValidSchema& schema);
 
+/**
+ *  Returns an encoder that encodes Avro standard for pretty printed JSON.
+ */
+AVRO_DECL EncoderPtr jsonPrettyEncoder(const ValidSchema& schema);
+
 }   // namespace avro
 
 #endif

http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonDom.cc
----------------------------------------------------------------------
diff --git a/lang/c++/impl/json/JsonDom.cc b/lang/c++/impl/json/JsonDom.cc
index 4c70b32..3f52f36 100644
--- a/lang/c++/impl/json/JsonDom.cc
+++ b/lang/c++/impl/json/JsonDom.cc
@@ -110,7 +110,7 @@ Entity loadEntity(const uint8_t* text, size_t len)
     return loadEntity(*in);
 }
 
-void writeEntity(JsonGenerator& g, const Entity& n)
+void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n)
 {
     switch (n.type()) {
     case etNull:
@@ -166,7 +166,7 @@ void Entity::ensureType(EntityType type) const
 std::string Entity::toString() const
 {
     std::auto_ptr<OutputStream> out = memoryOutputStream();
-    JsonGenerator g;
+    JsonGenerator<JsonNullFormatter> g;
     g.init(*out);
     writeEntity(g, *this);
     g.flush();

http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonDom.hh
----------------------------------------------------------------------
diff --git a/lang/c++/impl/json/JsonDom.hh b/lang/c++/impl/json/JsonDom.hh
index ae99d4f..7de4fbc 100644
--- a/lang/c++/impl/json/JsonDom.hh
+++ b/lang/c++/impl/json/JsonDom.hh
@@ -44,6 +44,9 @@ typedef std::vector<Entity> Array;
 typedef std::map<std::string, Entity> Object;
     
 class AVRO_DECL JsonParser;
+class JsonNullFormatter;
+
+template<typename F = JsonNullFormatter>
 class AVRO_DECL JsonGenerator;
 
 enum EntityType {
@@ -144,7 +147,7 @@ AVRO_DECL Entity loadEntity(InputStream& in);
 AVRO_DECL Entity loadEntity(const char* text);
 AVRO_DECL Entity loadEntity(const uint8_t* text, size_t len);
 
-void writeEntity(JsonGenerator& g, const Entity& n);
+void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n);
 
 }
 }

http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonIO.cc
----------------------------------------------------------------------
diff --git a/lang/c++/impl/json/JsonIO.cc b/lang/c++/impl/json/JsonIO.cc
index f21e822..2e7d82f 100644
--- a/lang/c++/impl/json/JsonIO.cc
+++ b/lang/c++/impl/json/JsonIO.cc
@@ -16,7 +16,6 @@
  * limitations under the License.
  */
 
-#include "boost/math/special_functions/fpclassify.hpp"
 #include "JsonIO.hh"
 
 namespace avro {
@@ -351,24 +350,6 @@ JsonParser::Token JsonParser::tryLiteral(const char exp[], size_t n,
Token tk)
     return tk;
 }
 
-void JsonGenerator::encodeNumber(double t) {
-    sep();
-    std::ostringstream oss;
-    if (boost::math::isfinite(t)) {
-        oss << t;
-    } else if (boost::math::isnan(t)) {
-        oss << "NaN";
-    } else if (t == std::numeric_limits<double>::infinity()) {
-        oss << "Infinity";
-    } else {
-        oss << "-Infinity";
-    }
-    const std::string& s = oss.str();
-    out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size());
-    sep2();
-}
-
-
 }
 }
 

http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonIO.hh
----------------------------------------------------------------------
diff --git a/lang/c++/impl/json/JsonIO.hh b/lang/c++/impl/json/JsonIO.hh
index 1406d7a..a5ada2d 100644
--- a/lang/c++/impl/json/JsonIO.hh
+++ b/lang/c++/impl/json/JsonIO.hh
@@ -23,6 +23,7 @@
 #include <stack>
 #include <string>
 #include <sstream>
+#include <boost/math/special_functions/fpclassify.hpp>
 #include <boost/utility.hpp>
 
 #include "Config.hh"
@@ -132,8 +133,58 @@ public:
     }
 };
 
+struct AVRO_DECL JsonNullFormatter {
+    JsonNullFormatter(StreamWriter&) { }
+
+    void handleObjectStart() {}
+    void handleObjectEnd() {}
+    void handleValueEnd() {}
+    void handleColon() {}
+};
+
+class AVRO_DECL JsonPrettyFormatter {
+    StreamWriter& out_;
+    size_t level_;
+    std::vector<uint8_t> indent_;
+
+    static const int CHARS_PER_LEVEL = 2;
+
+    void printIndent() {
+        size_t charsToIndent = level_ * CHARS_PER_LEVEL;
+        if (indent_.size() < charsToIndent) {
+            indent_.resize(charsToIndent * 2, ' ');
+        }
+        out_.writeBytes(&(indent_[0]), charsToIndent);
+    }
+public:
+    JsonPrettyFormatter(StreamWriter& out) : out_(out), level_(0), indent_(10, ' ') {
}
+
+    void handleObjectStart() {
+        out_.write('\n');
+        ++level_;
+        printIndent();
+    }
+
+    void handleObjectEnd() {
+        out_.write('\n');
+        --level_;
+        printIndent();
+    }
+
+    void handleValueEnd() {
+        out_.write('\n');
+        printIndent();
+    }
+
+    void handleColon() {
+        out_.write(' ');
+    }
+};
+
+template <class F>
 class AVRO_DECL JsonGenerator {
     StreamWriter out_;
+    F formatter_;
     enum State {
         stStart,
         stArray0,
@@ -210,6 +261,7 @@ class AVRO_DECL JsonGenerator {
     void sep() {
         if (top == stArrayN) {
             out_.write(',');
+            formatter_.handleValueEnd();
         } else if (top == stArray0) {
             top = stArrayN;
         }
@@ -222,7 +274,7 @@ class AVRO_DECL JsonGenerator {
     }
 
 public:
-    JsonGenerator() : top(stStart) { }
+    JsonGenerator() : formatter_(out_), top(stStart) { }
 
     void init(OutputStream& os) {
         out_.reset(os);
@@ -258,13 +310,30 @@ public:
         sep2();
     }
 
-    void encodeNumber(double t);
+    void encodeNumber(double t) {
+        sep();
+        std::ostringstream oss;
+        if (boost::math::isfinite(t)) {
+            oss << t;
+        } else if (boost::math::isnan(t)) {
+            oss << "NaN";
+        } else if (t == std::numeric_limits<double>::infinity()) {
+            oss << "Infinity";
+        } else {
+            oss << "-Infinity";
+        }
+        const std::string& s = oss.str();
+        out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size());
+        sep2();
+    }
+
 
     void encodeString(const std::string& s) {
         if (top == stMap0) {
             top = stKey;
         } else if (top == stMapN) {
             out_.write(',');
+            formatter_.handleValueEnd();
             top = stKey;
         } else if (top == stKey) {
             top = stMapN;
@@ -274,6 +343,7 @@ public:
         doEncodeString(s);
         if (top == stKey) {
             out_.write(':');
+            formatter_.handleColon();
         }
     }
 
@@ -293,11 +363,13 @@ public:
         stateStack.push(top);
         top = stArray0;
         out_.write('[');
+        formatter_.handleObjectStart();
     }
 
     void arrayEnd() {
         top = stateStack.top();
         stateStack.pop();
+        formatter_.handleObjectEnd();
         out_.write(']');
         sep2();
     }
@@ -307,11 +379,13 @@ public:
         stateStack.push(top);
         top = stMap0;
         out_.write('{');
+        formatter_.handleObjectStart();
     }
 
     void objectEnd() {
         top = stateStack.top();
         stateStack.pop();
+        formatter_.handleObjectEnd();
         out_.write('}');
         sep2();
     }

http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/parsing/JsonCodec.cc
----------------------------------------------------------------------
diff --git a/lang/c++/impl/parsing/JsonCodec.cc b/lang/c++/impl/parsing/JsonCodec.cc
index f18a1c9..ead44ca 100644
--- a/lang/c++/impl/parsing/JsonCodec.cc
+++ b/lang/c++/impl/parsing/JsonCodec.cc
@@ -54,6 +54,7 @@ using std::istringstream;
 
 using avro::json::JsonParser;
 using avro::json::JsonGenerator;
+using avro::json::JsonNullFormatter;
 
 class JsonGrammarGenerator : public ValidatingGrammarGenerator {
     ProductionPtr doGenerate(const NodePtr& n,
@@ -464,11 +465,11 @@ size_t JsonDecoder<P>::decodeUnionIndex()
     return result;
 }
 
-
+template<typename F = JsonNullFormatter>
 class JsonHandler {
-    JsonGenerator& generator_;
+    JsonGenerator<F>& generator_;
 public:
-    JsonHandler(JsonGenerator& g) : generator_(g) { }
+    JsonHandler(JsonGenerator<F>& g) : generator_(g) { }
     size_t handle(const Symbol& s) {
         switch (s.kind()) {
         case Symbol::sRecordStart:
@@ -487,10 +488,10 @@ public:
     }
 };
 
-template <typename P>
+template <typename P, typename F = JsonNullFormatter>
 class JsonEncoder : public Encoder {
-    JsonGenerator out_;
-    JsonHandler handler_;
+    JsonGenerator<F> out_;
+    JsonHandler<F> handler_;
     P parser_;
 
     void init(OutputStream& os);
@@ -518,49 +519,49 @@ public:
         parser_(JsonGrammarGenerator().generate(schema), NULL, handler_) { }
 };
 
-template<typename P>
-void JsonEncoder<P>::init(OutputStream& os)
+template<typename P, typename F>
+void JsonEncoder<P, F>::init(OutputStream& os)
 {
     out_.init(os);
 }
 
-template<typename P>
-void JsonEncoder<P>::flush()
+template<typename P, typename F>
+void JsonEncoder<P, F>::flush()
 {
     parser_.processImplicitActions();
     out_.flush();
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeNull()
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeNull()
 {
     parser_.advance(Symbol::sNull);
     out_.encodeNull();
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeBool(bool b)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeBool(bool b)
 {
     parser_.advance(Symbol::sBool);
     out_.encodeBool(b);
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeInt(int32_t i)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeInt(int32_t i)
 {
     parser_.advance(Symbol::sInt);
     out_.encodeNumber(i);
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeLong(int64_t l)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeLong(int64_t l)
 {
     parser_.advance(Symbol::sLong);
     out_.encodeNumber(l);
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeFloat(float f)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeFloat(float f)
 {
     parser_.advance(Symbol::sFloat);
     if (f == std::numeric_limits<float>::infinity()) {
@@ -574,8 +575,8 @@ void JsonEncoder<P>::encodeFloat(float f)
     }
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeDouble(double d)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeDouble(double d)
 {
     parser_.advance(Symbol::sDouble);
     if (d == std::numeric_limits<double>::infinity()) {
@@ -589,74 +590,74 @@ void JsonEncoder<P>::encodeDouble(double d)
     }
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeString(const std::string& s)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeString(const std::string& s)
 {
     parser_.advance(Symbol::sString);
     out_.encodeString(s);
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeBytes(const uint8_t *bytes, size_t len)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeBytes(const uint8_t *bytes, size_t len)
 {
     parser_.advance(Symbol::sBytes);
     out_.encodeBinary(bytes, len);
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeFixed(const uint8_t *bytes, size_t len)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeFixed(const uint8_t *bytes, size_t len)
 {
     parser_.advance(Symbol::sFixed);
     parser_.assertSize(len);
     out_.encodeBinary(bytes, len);
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeEnum(size_t e)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeEnum(size_t e)
 {
     parser_.advance(Symbol::sEnum);
     const string& s = parser_.nameForIndex(e);
     out_.encodeString(s);
 }
 
-template<typename P>
-void JsonEncoder<P>::arrayStart()
+template<typename P, typename F>
+void JsonEncoder<P, F>::arrayStart()
 {
     parser_.advance(Symbol::sArrayStart);
     out_.arrayStart();
 }
 
-template<typename P>
-void JsonEncoder<P>::arrayEnd()
+template<typename P, typename F>
+void JsonEncoder<P, F>::arrayEnd()
 {
     parser_.popRepeater();
     parser_.advance(Symbol::sArrayEnd);
     out_.arrayEnd();
 }
 
-template<typename P>
-void JsonEncoder<P>::mapStart()
+template<typename P, typename F>
+void JsonEncoder<P, F>::mapStart()
 {
     parser_.advance(Symbol::sMapStart);
     out_.objectStart();
 }
 
-template<typename P>
-void JsonEncoder<P>::mapEnd()
+template<typename P, typename F>
+void JsonEncoder<P, F>::mapEnd()
 {
     parser_.popRepeater();
     parser_.advance(Symbol::sMapEnd);
     out_.objectEnd();
 }
 
-template<typename P>
-void JsonEncoder<P>::setItemCount(size_t count)
+template<typename P, typename F>
+void JsonEncoder<P, F>::setItemCount(size_t count)
 {
     parser_.setRepeatCount(count);
 }
 
-template<typename P>
-void JsonEncoder<P>::startItem()
+template<typename P, typename F>
+void JsonEncoder<P, F>::startItem()
 {
     parser_.processImplicitActions();
     if (parser_.top() != Symbol::sRepeater) {
@@ -664,8 +665,8 @@ void JsonEncoder<P>::startItem()
     }
 }
 
-template<typename P>
-void JsonEncoder<P>::encodeUnionIndex(size_t e)
+template<typename P, typename F>
+void JsonEncoder<P, F>::encodeUnionIndex(size_t e)
 {
     parser_.advance(Symbol::sUnion);
 
@@ -689,7 +690,13 @@ DecoderPtr jsonDecoder(const ValidSchema& s)
 EncoderPtr jsonEncoder(const ValidSchema& schema)
 {
     return boost::make_shared<parsing::JsonEncoder<
-        parsing::SimpleParser<parsing::JsonHandler> > >(schema);
+        parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonNullFormatter>
>, avro::json::JsonNullFormatter> >(schema);
+}
+
+EncoderPtr jsonPrettyEncoder(const ValidSchema& schema)
+{
+    return boost::make_shared<parsing::JsonEncoder<
+        parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonPrettyFormatter>
>, avro::json::JsonPrettyFormatter> >(schema);
 }
 
 }   // namespace avro

http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/test/CodecTests.cc
----------------------------------------------------------------------
diff --git a/lang/c++/test/CodecTests.cc b/lang/c++/test/CodecTests.cc
index 61d29a2..c0ca1e0 100644
--- a/lang/c++/test/CodecTests.cc
+++ b/lang/c++/test/CodecTests.cc
@@ -60,7 +60,7 @@ static const unsigned int count = 10;
 
 /**
  * A bunch of tests that share quite a lot of infrastructure between them.
- * The basic idea is to generate avro data for according to a schema and
+ * The basic idea is to generate avro data according to a schema and
  * then read back and compare the data with the original. But quite a few
  * variations are possible:
  * 1. While reading back, one can skip different data elements
@@ -1385,6 +1385,15 @@ struct JsonCodec {
     }
 };
 
+struct JsonPrettyCodec {
+    static EncoderPtr newEncoder(const ValidSchema& schema) {
+        return jsonPrettyEncoder(schema);
+    }
+    static DecoderPtr newDecoder(const ValidSchema& schema) {
+        return jsonDecoder(schema);
+    }
+};
+
 struct BinaryEncoderResolvingDecoderFactory : public BinaryEncoderFactory {
     static DecoderPtr newDecoder(const ValidSchema& schema) {
         return resolvingDecoder(schema, schema, binaryDecoder());
@@ -1415,6 +1424,7 @@ void add_tests(boost::unit_test::test_suite& ts)
     ADD_TESTS(ts, BinaryCodecFactory, testCodec, data);
     ADD_TESTS(ts, ValidatingCodecFactory, testCodec, data);
     ADD_TESTS(ts, JsonCodec, testCodec, data);
+    ADD_TESTS(ts, JsonPrettyCodec, testCodec, data);
     ADD_TESTS(ts, BinaryEncoderResolvingDecoderFactory, testCodec, data);
     ADD_TESTS(ts, ValidatingCodecFactory, testReaderFail, data2);
     ADD_TESTS(ts, ValidatingCodecFactory, testWriterFail, data2);
@@ -1500,6 +1510,7 @@ static void testLimitsJsonCodec()
     "]}";
     ValidSchema schema = parsing::makeValidSchema(s);
     testLimits(jsonEncoder(schema), jsonDecoder(schema));
+    testLimits(jsonPrettyEncoder(schema), jsonDecoder(schema));
 }
 
 struct JsonData {
@@ -1522,7 +1533,6 @@ static void testJson(const JsonData& data)
 {
     ValidSchema schema = parsing::makeValidSchema(data.schema);
     EncoderPtr e = jsonEncoder(schema);
-    
 }
 
 }   // namespace avro


Mime
View raw message