Repository: avro Updated Branches: refs/heads/master 32067dec2 -> e98caa993
AVRO-1784. C++: Should have a json encoder that pretty prints Project: http://git-wip-us.apache.org/repos/asf/avro/repo Commit: http://git-wip-us.apache.org/repos/asf/avro/commit/e98caa99 Tree: http://git-wip-us.apache.org/repos/asf/avro/tree/e98caa99 Diff: http://git-wip-us.apache.org/repos/asf/avro/diff/e98caa99 Branch: refs/heads/master Commit: e98caa9937c47a4b44c80e0497995fd404102755 Parents: 32067de Author: Thiruvalluvan M. G <[email protected]> Authored: Wed Apr 13 10:53:29 2016 +0530 Committer: Thiruvalluvan M. G <[email protected]> Committed: Wed Apr 13 10:58:22 2016 +0530 ---------------------------------------------------------------------- CHANGES.txt | 2 + lang/c++/api/Encoder.hh | 7 ++- lang/c++/impl/json/JsonDom.cc | 4 +- lang/c++/impl/json/JsonDom.hh | 5 +- lang/c++/impl/json/JsonIO.cc | 19 ------- lang/c++/impl/json/JsonIO.hh | 78 +++++++++++++++++++++++++- lang/c++/impl/parsing/JsonCodec.cc | 97 ++++++++++++++++++--------------- lang/c++/test/CodecTests.cc | 14 ++++- 8 files changed, 154 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index b44745f..1399a94 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -89,6 +89,8 @@ Avro 1.8.0 (22 January 2016) AVRO-1767. JS: Fall back to unwrapped union values. (Matthieu Monsch via blue) + AVRO-1784. C++: Should have a json encoder that pretty prints. (John McClean via thiru) + OPTIMIZATIONS AVRO-1760. Java: Thread scalability problem with the use of SynchronizedMap http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/api/Encoder.hh ---------------------------------------------------------------------- diff --git a/lang/c++/api/Encoder.hh b/lang/c++/api/Encoder.hh index a5dce71..e13fa69 100644 --- a/lang/c++/api/Encoder.hh +++ b/lang/c++/api/Encoder.hh @@ -156,10 +156,15 @@ AVRO_DECL EncoderPtr validatingEncoder(const ValidSchema& schema, const EncoderPtr& base); /** - * Returns an encoder that can encode Avro standard for JSON. + * Returns an encoder that encodes Avro standard for JSON. */ AVRO_DECL EncoderPtr jsonEncoder(const ValidSchema& schema); +/** + * Returns an encoder that encodes Avro standard for pretty printed JSON. + */ +AVRO_DECL EncoderPtr jsonPrettyEncoder(const ValidSchema& schema); + } // namespace avro #endif http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonDom.cc ---------------------------------------------------------------------- diff --git a/lang/c++/impl/json/JsonDom.cc b/lang/c++/impl/json/JsonDom.cc index 4c70b32..3f52f36 100644 --- a/lang/c++/impl/json/JsonDom.cc +++ b/lang/c++/impl/json/JsonDom.cc @@ -110,7 +110,7 @@ Entity loadEntity(const uint8_t* text, size_t len) return loadEntity(*in); } -void writeEntity(JsonGenerator& g, const Entity& n) +void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n) { switch (n.type()) { case etNull: @@ -166,7 +166,7 @@ void Entity::ensureType(EntityType type) const std::string Entity::toString() const { std::auto_ptr<OutputStream> out = memoryOutputStream(); - JsonGenerator g; + JsonGenerator<JsonNullFormatter> g; g.init(*out); writeEntity(g, *this); g.flush(); http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonDom.hh ---------------------------------------------------------------------- diff --git a/lang/c++/impl/json/JsonDom.hh b/lang/c++/impl/json/JsonDom.hh index ae99d4f..7de4fbc 100644 --- a/lang/c++/impl/json/JsonDom.hh +++ b/lang/c++/impl/json/JsonDom.hh @@ -44,6 +44,9 @@ typedef std::vector<Entity> Array; typedef std::map<std::string, Entity> Object; class AVRO_DECL JsonParser; +class JsonNullFormatter; + +template<typename F = JsonNullFormatter> class AVRO_DECL JsonGenerator; enum EntityType { @@ -144,7 +147,7 @@ AVRO_DECL Entity loadEntity(InputStream& in); AVRO_DECL Entity loadEntity(const char* text); AVRO_DECL Entity loadEntity(const uint8_t* text, size_t len); -void writeEntity(JsonGenerator& g, const Entity& n); +void writeEntity(JsonGenerator<JsonNullFormatter>& g, const Entity& n); } } http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonIO.cc ---------------------------------------------------------------------- diff --git a/lang/c++/impl/json/JsonIO.cc b/lang/c++/impl/json/JsonIO.cc index f21e822..2e7d82f 100644 --- a/lang/c++/impl/json/JsonIO.cc +++ b/lang/c++/impl/json/JsonIO.cc @@ -16,7 +16,6 @@ * limitations under the License. */ -#include "boost/math/special_functions/fpclassify.hpp" #include "JsonIO.hh" namespace avro { @@ -351,24 +350,6 @@ JsonParser::Token JsonParser::tryLiteral(const char exp[], size_t n, Token tk) return tk; } -void JsonGenerator::encodeNumber(double t) { - sep(); - std::ostringstream oss; - if (boost::math::isfinite(t)) { - oss << t; - } else if (boost::math::isnan(t)) { - oss << "NaN"; - } else if (t == std::numeric_limits<double>::infinity()) { - oss << "Infinity"; - } else { - oss << "-Infinity"; - } - const std::string& s = oss.str(); - out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size()); - sep2(); -} - - } } http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/json/JsonIO.hh ---------------------------------------------------------------------- diff --git a/lang/c++/impl/json/JsonIO.hh b/lang/c++/impl/json/JsonIO.hh index 1406d7a..a5ada2d 100644 --- a/lang/c++/impl/json/JsonIO.hh +++ b/lang/c++/impl/json/JsonIO.hh @@ -23,6 +23,7 @@ #include <stack> #include <string> #include <sstream> +#include <boost/math/special_functions/fpclassify.hpp> #include <boost/utility.hpp> #include "Config.hh" @@ -132,8 +133,58 @@ public: } }; +struct AVRO_DECL JsonNullFormatter { + JsonNullFormatter(StreamWriter&) { } + + void handleObjectStart() {} + void handleObjectEnd() {} + void handleValueEnd() {} + void handleColon() {} +}; + +class AVRO_DECL JsonPrettyFormatter { + StreamWriter& out_; + size_t level_; + std::vector<uint8_t> indent_; + + static const int CHARS_PER_LEVEL = 2; + + void printIndent() { + size_t charsToIndent = level_ * CHARS_PER_LEVEL; + if (indent_.size() < charsToIndent) { + indent_.resize(charsToIndent * 2, ' '); + } + out_.writeBytes(&(indent_[0]), charsToIndent); + } +public: + JsonPrettyFormatter(StreamWriter& out) : out_(out), level_(0), indent_(10, ' ') { } + + void handleObjectStart() { + out_.write('\n'); + ++level_; + printIndent(); + } + + void handleObjectEnd() { + out_.write('\n'); + --level_; + printIndent(); + } + + void handleValueEnd() { + out_.write('\n'); + printIndent(); + } + + void handleColon() { + out_.write(' '); + } +}; + +template <class F> class AVRO_DECL JsonGenerator { StreamWriter out_; + F formatter_; enum State { stStart, stArray0, @@ -210,6 +261,7 @@ class AVRO_DECL JsonGenerator { void sep() { if (top == stArrayN) { out_.write(','); + formatter_.handleValueEnd(); } else if (top == stArray0) { top = stArrayN; } @@ -222,7 +274,7 @@ class AVRO_DECL JsonGenerator { } public: - JsonGenerator() : top(stStart) { } + JsonGenerator() : formatter_(out_), top(stStart) { } void init(OutputStream& os) { out_.reset(os); @@ -258,13 +310,30 @@ public: sep2(); } - void encodeNumber(double t); + void encodeNumber(double t) { + sep(); + std::ostringstream oss; + if (boost::math::isfinite(t)) { + oss << t; + } else if (boost::math::isnan(t)) { + oss << "NaN"; + } else if (t == std::numeric_limits<double>::infinity()) { + oss << "Infinity"; + } else { + oss << "-Infinity"; + } + const std::string& s = oss.str(); + out_.writeBytes(reinterpret_cast<const uint8_t*>(&s[0]), s.size()); + sep2(); + } + void encodeString(const std::string& s) { if (top == stMap0) { top = stKey; } else if (top == stMapN) { out_.write(','); + formatter_.handleValueEnd(); top = stKey; } else if (top == stKey) { top = stMapN; @@ -274,6 +343,7 @@ public: doEncodeString(s); if (top == stKey) { out_.write(':'); + formatter_.handleColon(); } } @@ -293,11 +363,13 @@ public: stateStack.push(top); top = stArray0; out_.write('['); + formatter_.handleObjectStart(); } void arrayEnd() { top = stateStack.top(); stateStack.pop(); + formatter_.handleObjectEnd(); out_.write(']'); sep2(); } @@ -307,11 +379,13 @@ public: stateStack.push(top); top = stMap0; out_.write('{'); + formatter_.handleObjectStart(); } void objectEnd() { top = stateStack.top(); stateStack.pop(); + formatter_.handleObjectEnd(); out_.write('}'); sep2(); } http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/impl/parsing/JsonCodec.cc ---------------------------------------------------------------------- diff --git a/lang/c++/impl/parsing/JsonCodec.cc b/lang/c++/impl/parsing/JsonCodec.cc index f18a1c9..ead44ca 100644 --- a/lang/c++/impl/parsing/JsonCodec.cc +++ b/lang/c++/impl/parsing/JsonCodec.cc @@ -54,6 +54,7 @@ using std::istringstream; using avro::json::JsonParser; using avro::json::JsonGenerator; +using avro::json::JsonNullFormatter; class JsonGrammarGenerator : public ValidatingGrammarGenerator { ProductionPtr doGenerate(const NodePtr& n, @@ -464,11 +465,11 @@ size_t JsonDecoder<P>::decodeUnionIndex() return result; } - +template<typename F = JsonNullFormatter> class JsonHandler { - JsonGenerator& generator_; + JsonGenerator<F>& generator_; public: - JsonHandler(JsonGenerator& g) : generator_(g) { } + JsonHandler(JsonGenerator<F>& g) : generator_(g) { } size_t handle(const Symbol& s) { switch (s.kind()) { case Symbol::sRecordStart: @@ -487,10 +488,10 @@ public: } }; -template <typename P> +template <typename P, typename F = JsonNullFormatter> class JsonEncoder : public Encoder { - JsonGenerator out_; - JsonHandler handler_; + JsonGenerator<F> out_; + JsonHandler<F> handler_; P parser_; void init(OutputStream& os); @@ -518,49 +519,49 @@ public: parser_(JsonGrammarGenerator().generate(schema), NULL, handler_) { } }; -template<typename P> -void JsonEncoder<P>::init(OutputStream& os) +template<typename P, typename F> +void JsonEncoder<P, F>::init(OutputStream& os) { out_.init(os); } -template<typename P> -void JsonEncoder<P>::flush() +template<typename P, typename F> +void JsonEncoder<P, F>::flush() { parser_.processImplicitActions(); out_.flush(); } -template<typename P> -void JsonEncoder<P>::encodeNull() +template<typename P, typename F> +void JsonEncoder<P, F>::encodeNull() { parser_.advance(Symbol::sNull); out_.encodeNull(); } -template<typename P> -void JsonEncoder<P>::encodeBool(bool b) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeBool(bool b) { parser_.advance(Symbol::sBool); out_.encodeBool(b); } -template<typename P> -void JsonEncoder<P>::encodeInt(int32_t i) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeInt(int32_t i) { parser_.advance(Symbol::sInt); out_.encodeNumber(i); } -template<typename P> -void JsonEncoder<P>::encodeLong(int64_t l) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeLong(int64_t l) { parser_.advance(Symbol::sLong); out_.encodeNumber(l); } -template<typename P> -void JsonEncoder<P>::encodeFloat(float f) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeFloat(float f) { parser_.advance(Symbol::sFloat); if (f == std::numeric_limits<float>::infinity()) { @@ -574,8 +575,8 @@ void JsonEncoder<P>::encodeFloat(float f) } } -template<typename P> -void JsonEncoder<P>::encodeDouble(double d) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeDouble(double d) { parser_.advance(Symbol::sDouble); if (d == std::numeric_limits<double>::infinity()) { @@ -589,74 +590,74 @@ void JsonEncoder<P>::encodeDouble(double d) } } -template<typename P> -void JsonEncoder<P>::encodeString(const std::string& s) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeString(const std::string& s) { parser_.advance(Symbol::sString); out_.encodeString(s); } -template<typename P> -void JsonEncoder<P>::encodeBytes(const uint8_t *bytes, size_t len) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeBytes(const uint8_t *bytes, size_t len) { parser_.advance(Symbol::sBytes); out_.encodeBinary(bytes, len); } -template<typename P> -void JsonEncoder<P>::encodeFixed(const uint8_t *bytes, size_t len) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeFixed(const uint8_t *bytes, size_t len) { parser_.advance(Symbol::sFixed); parser_.assertSize(len); out_.encodeBinary(bytes, len); } -template<typename P> -void JsonEncoder<P>::encodeEnum(size_t e) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeEnum(size_t e) { parser_.advance(Symbol::sEnum); const string& s = parser_.nameForIndex(e); out_.encodeString(s); } -template<typename P> -void JsonEncoder<P>::arrayStart() +template<typename P, typename F> +void JsonEncoder<P, F>::arrayStart() { parser_.advance(Symbol::sArrayStart); out_.arrayStart(); } -template<typename P> -void JsonEncoder<P>::arrayEnd() +template<typename P, typename F> +void JsonEncoder<P, F>::arrayEnd() { parser_.popRepeater(); parser_.advance(Symbol::sArrayEnd); out_.arrayEnd(); } -template<typename P> -void JsonEncoder<P>::mapStart() +template<typename P, typename F> +void JsonEncoder<P, F>::mapStart() { parser_.advance(Symbol::sMapStart); out_.objectStart(); } -template<typename P> -void JsonEncoder<P>::mapEnd() +template<typename P, typename F> +void JsonEncoder<P, F>::mapEnd() { parser_.popRepeater(); parser_.advance(Symbol::sMapEnd); out_.objectEnd(); } -template<typename P> -void JsonEncoder<P>::setItemCount(size_t count) +template<typename P, typename F> +void JsonEncoder<P, F>::setItemCount(size_t count) { parser_.setRepeatCount(count); } -template<typename P> -void JsonEncoder<P>::startItem() +template<typename P, typename F> +void JsonEncoder<P, F>::startItem() { parser_.processImplicitActions(); if (parser_.top() != Symbol::sRepeater) { @@ -664,8 +665,8 @@ void JsonEncoder<P>::startItem() } } -template<typename P> -void JsonEncoder<P>::encodeUnionIndex(size_t e) +template<typename P, typename F> +void JsonEncoder<P, F>::encodeUnionIndex(size_t e) { parser_.advance(Symbol::sUnion); @@ -689,7 +690,13 @@ DecoderPtr jsonDecoder(const ValidSchema& s) EncoderPtr jsonEncoder(const ValidSchema& schema) { return boost::make_shared<parsing::JsonEncoder< - parsing::SimpleParser<parsing::JsonHandler> > >(schema); + parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonNullFormatter> >, avro::json::JsonNullFormatter> >(schema); +} + +EncoderPtr jsonPrettyEncoder(const ValidSchema& schema) +{ + return boost::make_shared<parsing::JsonEncoder< + parsing::SimpleParser<parsing::JsonHandler<avro::json::JsonPrettyFormatter> >, avro::json::JsonPrettyFormatter> >(schema); } } // namespace avro http://git-wip-us.apache.org/repos/asf/avro/blob/e98caa99/lang/c++/test/CodecTests.cc ---------------------------------------------------------------------- diff --git a/lang/c++/test/CodecTests.cc b/lang/c++/test/CodecTests.cc index 61d29a2..c0ca1e0 100644 --- a/lang/c++/test/CodecTests.cc +++ b/lang/c++/test/CodecTests.cc @@ -60,7 +60,7 @@ static const unsigned int count = 10; /** * A bunch of tests that share quite a lot of infrastructure between them. - * The basic idea is to generate avro data for according to a schema and + * The basic idea is to generate avro data according to a schema and * then read back and compare the data with the original. But quite a few * variations are possible: * 1. While reading back, one can skip different data elements @@ -1385,6 +1385,15 @@ struct JsonCodec { } }; +struct JsonPrettyCodec { + static EncoderPtr newEncoder(const ValidSchema& schema) { + return jsonPrettyEncoder(schema); + } + static DecoderPtr newDecoder(const ValidSchema& schema) { + return jsonDecoder(schema); + } +}; + struct BinaryEncoderResolvingDecoderFactory : public BinaryEncoderFactory { static DecoderPtr newDecoder(const ValidSchema& schema) { return resolvingDecoder(schema, schema, binaryDecoder()); @@ -1415,6 +1424,7 @@ void add_tests(boost::unit_test::test_suite& ts) ADD_TESTS(ts, BinaryCodecFactory, testCodec, data); ADD_TESTS(ts, ValidatingCodecFactory, testCodec, data); ADD_TESTS(ts, JsonCodec, testCodec, data); + ADD_TESTS(ts, JsonPrettyCodec, testCodec, data); ADD_TESTS(ts, BinaryEncoderResolvingDecoderFactory, testCodec, data); ADD_TESTS(ts, ValidatingCodecFactory, testReaderFail, data2); ADD_TESTS(ts, ValidatingCodecFactory, testWriterFail, data2); @@ -1500,6 +1510,7 @@ static void testLimitsJsonCodec() "]}"; ValidSchema schema = parsing::makeValidSchema(s); testLimits(jsonEncoder(schema), jsonDecoder(schema)); + testLimits(jsonPrettyEncoder(schema), jsonDecoder(schema)); } struct JsonData { @@ -1522,7 +1533,6 @@ static void testJson(const JsonData& data) { ValidSchema schema = parsing::makeValidSchema(data.schema); EncoderPtr e = jsonEncoder(schema); - } } // namespace avro
