/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #define __STDC_LIMIT_MACROS #include #include #include #include #include #include #include #include #include "ValidatingCodec.hh" #include "Symbol.hh" #include "ValidSchema.hh" #include "Decoder.hh" #include "Encoder.hh" #include "NodeImpl.hh" #include "../json/JsonIO.hh" namespace rmf_avro { namespace parsing { using boost::shared_ptr; using boost::static_pointer_cast; using std::map; using std::vector; using std::string; using std::reverse; using std::ostringstream; using std::istringstream; using rmf_avro::json::JsonParser; using rmf_avro::json::JsonGenerator; class JsonGrammarGenerator : public ValidatingGrammarGenerator { Production doGenerate(const NodePtr& n, std::map > &m); }; static std::string nameOf(const NodePtr& n) { if (n->hasName()) { return n->name(); } std::ostringstream oss; oss << n->type(); return oss.str(); } Production JsonGrammarGenerator::doGenerate(const NodePtr& n, std::map > &m) { switch (n->type()) { case AVRO_NULL: case AVRO_BOOL: case AVRO_INT: case AVRO_LONG: case AVRO_FLOAT: case AVRO_DOUBLE: case AVRO_STRING: case AVRO_BYTES: case AVRO_FIXED: case AVRO_ARRAY: case AVRO_MAP: case AVRO_SYMBOLIC: return ValidatingGrammarGenerator::doGenerate(n, m); case AVRO_RECORD: { Production result; m.erase(n); size_t c = n->leaves(); result.reserve(2 + 2 * c); result.push_back(Symbol::recordStartSymbol()); for (size_t i = 0; i < c; ++i) { const NodePtr& leaf = n->leafAt(i); Production v = doGenerate(leaf, m); result.push_back(Symbol::fieldSymbol(n->nameAt(i))); copy(v.rbegin(), v.rend(), back_inserter(result)); } result.push_back(Symbol::recordEndSymbol()); reverse(result.begin(), result.end()); bool found = m.find(n) != m.end(); shared_ptr p = boost::make_shared(result); m[n] = p; return found ? Production(1, Symbol::indirect(p)) : result; } case AVRO_ENUM: { vector nn; size_t c = n->names(); nn.reserve(c); for (size_t i = 0; i < c; ++i) { nn.push_back(n->nameAt(i)); } Symbol r[] = { Symbol::nameListSymbol(nn), Symbol::enumSymbol() }; Production result(r, r + 2); m[n] = boost::make_shared(result); return result; } case AVRO_UNION: { size_t c = n->leaves(); vector vv; vv.reserve(c); vector names; names.reserve(c); for (size_t i = 0; i < c; ++i) { const NodePtr& nn = n->leafAt(i); Production v = doGenerate(nn, m); if (nn->type() != AVRO_NULL) { Production v2; v2.push_back(Symbol::recordEndSymbol()); copy(v.begin(), v.end(), back_inserter(v2)); v.swap(v2); } vv.push_back(v); names.push_back(nameOf(nn)); } Symbol r[] = { Symbol::alternative(vv), Symbol::nameListSymbol(names), Symbol::unionSymbol() }; return Production(r, r + 3); } default: throw Exception("Unknown node type"); } } static void expectToken(JsonParser& in, JsonParser::Token tk) { JsonParser::Token tka = in.advance(); // handle doubles as long if (tka != tk && (tk != JsonParser::tkDouble || tka != JsonParser::tkLong)) { ostringstream oss; oss << "Incorrect token in the stream. Expected: " << JsonParser::toString(tk) << ", found " << JsonParser::toString(in.cur()); throw Exception(oss.str()); } } class JsonDecoderHandler { JsonParser& in_; public: JsonDecoderHandler(JsonParser& p) : in_(p) { } size_t handle(const Symbol& s) { switch (s.kind()) { case Symbol::sRecordStart: expectToken(in_, JsonParser::tkObjectStart); break; case Symbol::sRecordEnd: expectToken(in_, JsonParser::tkObjectEnd); break; case Symbol::sField: expectToken(in_, JsonParser::tkString); if (s.extra() != in_.stringValue()) { throw Exception("Incorrect field"); } break; default: break; } return 0; } }; template class JsonDecoder : public Decoder { JsonParser in_; JsonDecoderHandler handler_; P parser_; void init(InputStream& is); void decodeNull(); bool decodeBool(); int32_t decodeInt(); int64_t decodeLong(); float decodeFloat(); double decodeDouble(); void decodeString(string& value); void skipString(); void decodeBytes(vector& value); void skipBytes(); void decodeFixed(size_t n, vector& value); void skipFixed(size_t n); size_t decodeEnum(); size_t arrayStart(); size_t arrayNext(); size_t skipArray(); size_t mapStart(); size_t mapNext(); size_t skipMap(); size_t decodeUnionIndex(); void expect(JsonParser::Token tk); void skipComposite(); public: JsonDecoder(const ValidSchema& s) : handler_(in_), parser_(JsonGrammarGenerator().generate(s), NULL, handler_) { } }; template void JsonDecoder

::init(InputStream& is) { in_.init(is); } template void JsonDecoder

::expect(JsonParser::Token tk) { expectToken(in_, tk); } template void JsonDecoder

::decodeNull() { parser_.advance(Symbol::sNull); expect(JsonParser::tkNull); } template bool JsonDecoder

::decodeBool() { parser_.advance(Symbol::sBool); expect(JsonParser::tkBool); bool result = in_.boolValue(); return result; } template int32_t JsonDecoder

::decodeInt() { parser_.advance(Symbol::sInt); expect(JsonParser::tkLong); int64_t result = in_.longValue(); if (result < INT32_MIN || result > INT32_MAX) { throw Exception(boost::format("Value out of range for Avro int: %1%") % result); } return static_cast(result); } template int64_t JsonDecoder

::decodeLong() { parser_.advance(Symbol::sLong); expect(JsonParser::tkLong); int64_t result = in_.longValue(); return result; } template float JsonDecoder

::decodeFloat() { parser_.advance(Symbol::sFloat); expect(JsonParser::tkDouble); double result = in_.doubleValue(); return static_cast(result); } template double JsonDecoder

::decodeDouble() { parser_.advance(Symbol::sDouble); expect(JsonParser::tkDouble); double result = in_.doubleValue(); return result; } template void JsonDecoder

::decodeString(string& value) { parser_.advance(Symbol::sString); expect(JsonParser::tkString); value = in_.stringValue(); } template void JsonDecoder

::skipString() { parser_.advance(Symbol::sString); expect(JsonParser::tkString); } static vector toBytes(const string& s) { return vector(s.begin(), s.end()); } template void JsonDecoder

::decodeBytes(vector& value ) { parser_.advance(Symbol::sBytes); expect(JsonParser::tkString); value = toBytes(in_.stringValue()); } template void JsonDecoder

::skipBytes() { parser_.advance(Symbol::sBytes); expect(JsonParser::tkString); } template void JsonDecoder

::decodeFixed(size_t n, vector& value) { parser_.advance(Symbol::sFixed); parser_.assertSize(n); expect(JsonParser::tkString); value = toBytes(in_.stringValue()); if (value.size() != n) { throw Exception("Incorrect value for fixed"); } } template void JsonDecoder

::skipFixed(size_t n) { parser_.advance(Symbol::sFixed); parser_.assertSize(n); expect(JsonParser::tkString); vector result = toBytes(in_.stringValue()); if (result.size() != n) { throw Exception("Incorrect value for fixed"); } } template size_t JsonDecoder

::decodeEnum() { parser_.advance(Symbol::sEnum); expect(JsonParser::tkString); size_t result = parser_.indexForName(in_.stringValue()); return result; } template size_t JsonDecoder

::arrayStart() { parser_.advance(Symbol::sArrayStart); expect(JsonParser::tkArrayStart); return arrayNext(); } template size_t JsonDecoder

::arrayNext() { parser_.processImplicitActions(); if (in_.peek() == JsonParser::tkArrayEnd) { in_.advance(); parser_.popRepeater(); parser_.advance(Symbol::sArrayEnd); return 0; } parser_.setRepeatCount(1); return 1; } template void JsonDecoder

::skipComposite() { size_t level = 0; for (; ;) { switch (in_.advance()) { case JsonParser::tkArrayStart: case JsonParser::tkObjectStart: ++level; continue; case JsonParser::tkArrayEnd: case JsonParser::tkObjectEnd: if (level == 0) { return; } --level; continue; default: continue; } } } template size_t JsonDecoder

::skipArray() { parser_.advance(Symbol::sArrayStart); parser_.pop(); parser_.advance(Symbol::sArrayEnd); expect(JsonParser::tkArrayStart); skipComposite(); return 0; } template size_t JsonDecoder

::mapStart() { parser_.advance(Symbol::sMapStart); expect(JsonParser::tkObjectStart); return mapNext(); } template size_t JsonDecoder

::mapNext() { parser_.processImplicitActions(); if (in_.peek() == JsonParser::tkObjectEnd) { in_.advance(); parser_.popRepeater(); parser_.advance(Symbol::sMapEnd); return 0; } parser_.setRepeatCount(1); return 1; } template size_t JsonDecoder

::skipMap() { parser_.advance(Symbol::sMapStart); parser_.pop(); parser_.advance(Symbol::sMapEnd); expect(JsonParser::tkObjectStart); skipComposite(); return 0; } template size_t JsonDecoder

::decodeUnionIndex() { parser_.advance(Symbol::sUnion); size_t result; if (in_.peek() == JsonParser::tkNull) { result = parser_.indexForName("null"); } else { expect(JsonParser::tkObjectStart); expect(JsonParser::tkString); result = parser_.indexForName(in_.stringValue()); } parser_.selectBranch(result); return result; } class JsonHandler { JsonGenerator& generator_; public: JsonHandler(JsonGenerator& g) : generator_(g) { } size_t handle(const Symbol& s) { switch (s.kind()) { case Symbol::sRecordStart: generator_.objectStart(); break; case Symbol::sRecordEnd: generator_.objectEnd(); break; case Symbol::sField: generator_.encodeString(s.extra()); break; default: break; } return 0; } }; template class JsonEncoder : public Encoder { JsonGenerator out_; JsonHandler handler_; P parser_; void init(OutputStream& os); void flush(); void encodeNull(); void encodeBool(bool b); void encodeInt(int32_t i); void encodeLong(int64_t l); void encodeFloat(float f); void encodeDouble(double d); void encodeString(const std::string& s); void encodeBytes(const uint8_t *bytes, size_t len); void encodeFixed(const uint8_t *bytes, size_t len); void encodeEnum(size_t e); void arrayStart(); void arrayEnd(); void mapStart(); void mapEnd(); void setItemCount(size_t count); void startItem(); void encodeUnionIndex(size_t e); public: JsonEncoder(const ValidSchema& schema) : handler_(out_), parser_(JsonGrammarGenerator().generate(schema), NULL, handler_) { } }; template void JsonEncoder

::init(OutputStream& os) { out_.init(os); } template void JsonEncoder

::flush() { parser_.processImplicitActions(); out_.flush(); } template void JsonEncoder

::encodeNull() { parser_.advance(Symbol::sNull); out_.encodeNull(); } template void JsonEncoder

::encodeBool(bool b) { parser_.advance(Symbol::sBool); out_.encodeBool(b); } template void JsonEncoder

::encodeInt(int32_t i) { parser_.advance(Symbol::sInt); out_.encodeNumber(i); } template void JsonEncoder

::encodeLong(int64_t l) { parser_.advance(Symbol::sLong); out_.encodeNumber(l); } template void JsonEncoder

::encodeFloat(float f) { parser_.advance(Symbol::sFloat); out_.encodeNumber(f); } template void JsonEncoder

::encodeDouble(double d) { parser_.advance(Symbol::sDouble); out_.encodeNumber(d); } template void JsonEncoder

::encodeString(const std::string& s) { parser_.advance(Symbol::sString); out_.encodeString(s); } template void JsonEncoder

::encodeBytes(const uint8_t *bytes, size_t len) { parser_.advance(Symbol::sBytes); out_.encodeBinary(bytes, len); } template void JsonEncoder

::encodeFixed(const uint8_t *bytes, size_t len) { parser_.advance(Symbol::sFixed); parser_.assertSize(len); out_.encodeBinary(bytes, len); } template void JsonEncoder

::encodeEnum(size_t e) { parser_.advance(Symbol::sEnum); const string& s = parser_.nameForIndex(e); out_.encodeString(s); } template void JsonEncoder

::arrayStart() { parser_.advance(Symbol::sArrayStart); out_.arrayStart(); } template void JsonEncoder

::arrayEnd() { parser_.popRepeater(); parser_.advance(Symbol::sArrayEnd); out_.arrayEnd(); } template void JsonEncoder

::mapStart() { parser_.advance(Symbol::sMapStart); out_.objectStart(); } template void JsonEncoder

::mapEnd() { parser_.popRepeater(); parser_.advance(Symbol::sMapEnd); out_.objectEnd(); } template void JsonEncoder

::setItemCount(size_t count) { parser_.setRepeatCount(count); } template void JsonEncoder

::startItem() { parser_.processImplicitActions(); if (parser_.top() != Symbol::sRepeater) { throw Exception("startItem at not an item boundary"); } } template void JsonEncoder

::encodeUnionIndex(size_t e) { parser_.advance(Symbol::sUnion); const std::string name = parser_.nameForIndex(e); if (name != "null") { out_.objectStart(); out_.encodeString(name); } parser_.selectBranch(e); } } // namespace parsing DecoderPtr jsonDecoder(const ValidSchema& s) { return boost::make_shared > >(s); } EncoderPtr jsonEncoder(const ValidSchema& schema) { return boost::make_shared > >(schema); } } // namespace rmf_avro